From 742091b99ab9d102964b437123f04289b9e3ca00 Mon Sep 17 00:00:00 2001 From: kid Date: Mon, 20 Jul 2026 21:40:34 +0800 Subject: [PATCH 1/4] fix: generate_series overflow panics at i64 boundary and out-of-range dates Two overflow fixes in the generate_series/range table functions: - Integer series: advancing past i64::MAX (or i64::MIN for negative steps) panicked in debug builds and silently wrapped in release. advance() now uses checked_add and clamps the series end so iteration stops after the last reachable value, matching PostgreSQL/DuckDB (e.g. generate_series(9223372036854775806, 9223372036854775807, 2) returns one row 9223372036854775806). - Date series: converting Date32 days to timestamp nanoseconds used an unchecked multiplication, panicking at planning time for dates outside the nanosecond timestamp range. It now returns a planning error. Supersedes #22250 (closed unmerged); the integer-side approach follows the one approved there. Closes #22208, closes #22193. --- .../functions-table/src/generate_series.rs | 52 ++++++++++++++++--- .../test_files/table_functions.slt | 37 +++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index 0e8eca6bc2561..3f669fd94999d 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -27,7 +27,7 @@ use async_trait::async_trait; use datafusion_catalog::TableFunctionImpl; use datafusion_catalog::TableProvider; use datafusion_catalog::{Session, TableFunctionArgs}; -use datafusion_common::{Result, ScalarValue, plan_err}; +use datafusion_common::{Result, ScalarValue, plan_datafusion_err, plan_err}; use datafusion_expr::{Expr, TableType}; use datafusion_physical_expr::PhysicalSortExpr; use datafusion_physical_expr::expressions::Column; @@ -80,7 +80,12 @@ pub trait SeriesValue: fmt::Debug + Clone + Send + Sync + 'static { fn should_stop(&self, end: Self, step: &Self::StepType, include_end: bool) -> bool; /// Advance to the next value in the series - fn advance(&mut self, step: &Self::StepType) -> Result<()>; + /// + /// If advancing would overflow the value range, `end` is updated so that + /// the series terminates after the current value (matching the behavior + /// of PostgreSQL and DuckDB, which return the reachable values instead of + /// erroring). + fn advance(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()>; /// Create an Arrow array from a vector of values fn create_array(&self, values: Vec) -> Result; @@ -100,8 +105,19 @@ impl SeriesValue for i64 { reach_end_int64(*self, end, *step, include_end) } - fn advance(&mut self, step: &Self::StepType) -> Result<()> { - *self += step; + fn advance(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()> { + if let Some(next) = self.checked_add(*step) { + *self = next; + } else { + // Advancing would overflow: clamp `end` so the series stops after + // the current (last reachable) value instead of panicking or + // wrapping around. + *end = if *step > 0 { + self.saturating_sub(1) + } else { + self.saturating_add(1) + }; + } Ok(()) } @@ -155,7 +171,7 @@ impl SeriesValue for TimestampValue { } } - fn advance(&mut self, step: &Self::StepType) -> Result<()> { + fn advance(&mut self, _end: &mut Self, step: &Self::StepType) -> Result<()> { let tz = self .parsed_tz .unwrap_or_else(|| Tz::from_str("+00:00").unwrap()); @@ -417,7 +433,15 @@ impl LazyBatchGenerator for GenericSeriesState { .should_stop(self.end.clone(), &self.step, self.include_end) { buf.push(self.current.to_value_type()); - self.current.advance(&self.step)?; + if self + .current + .should_stop(self.end.clone(), &self.step, false) + { + self.current.advance(&mut self.end, &self.step)?; + break; + } + + self.current.advance(&mut self.end, &self.step)?; } if buf.is_empty() { @@ -740,8 +764,20 @@ impl GenerateSeriesFuncImpl { // Date32 is days since 1970-01-01, so multiply by nanoseconds per day const NANOS_PER_DAY: i64 = 24 * 60 * 60 * 1_000_000_000; - let start_ts = start_date as i64 * NANOS_PER_DAY; - let end_ts = end_date as i64 * NANOS_PER_DAY; + // Dates outside the nanosecond timestamp range (1677-09-21 to + // 2262-04-11) cannot be represented; return an error instead of + // panicking (debug) or silently wrapping (release). + let date_to_ts_nanos = |date: i32, arg: &str| { + (date as i64).checked_mul(NANOS_PER_DAY).ok_or_else(|| { + plan_datafusion_err!( + "{arg} for {} is out of range of nanosecond timestamps", + self.name + ) + }) + }; + + let start_ts = date_to_ts_nanos(start_date, "First argument")?; + let end_ts = date_to_ts_nanos(end_date, "Second argument")?; // Validate step interval validate_interval_step(step_interval)?; diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.slt index e1ab444d81044..c2c8ce24fa081 100644 --- a/datafusion/sqllogictest/test_files/table_functions.slt +++ b/datafusion/sqllogictest/test_files/table_functions.slt @@ -197,6 +197,43 @@ SELECT * FROM generate_series(1, 2, 3, 4) statement error DataFusion error: Error during planning: Argument \#1 must be an INTEGER, TIMESTAMP, DATE or NULL, got Utf8 SELECT * FROM generate_series('foo', 'bar') +# Regression test for https://github.com/apache/datafusion/issues/22208 +# A step that would overflow i64 after the last reachable value must return the +# reachable values instead of panicking, matching PostgreSQL/DuckDB behavior. +query I +SELECT * FROM generate_series(9223372036854775806, 9223372036854775807, 2) +---- +9223372036854775806 + +# Same, in the descending direction +query I +SELECT * FROM generate_series(-9223372036854775806, -9223372036854775808, -2) +---- +-9223372036854775806 +-9223372036854775808 + +# Landing exactly on i64::MAX must include it +query I +SELECT * FROM generate_series(9223372036854775805, 9223372036854775807, 2) +---- +9223372036854775805 +9223372036854775807 + +# Same overflow behavior for `range` (end exclusive) +query I +SELECT * FROM range(9223372036854775806, 9223372036854775807, 2) +---- +9223372036854775806 + +# Regression test for https://github.com/apache/datafusion/issues/22193 +# Dates outside the nanosecond timestamp range must produce a clean planning +# error instead of panicking (debug) or silently wrapping (release). +statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps +SELECT * FROM generate_series(DATE '0001-01-01', DATE '2000-01-01', INTERVAL '1' DAY) + +statement error DataFusion error: Error during planning: Second argument for generate_series is out of range of nanosecond timestamps +SELECT * FROM generate_series(DATE '2000-01-01', DATE '3000-01-01', INTERVAL '1' DAY) + # UDF and UDTF `generate_series` can be used simultaneously query ? rowsort SELECT generate_series(1, t1.end) FROM generate_series(3, 5) as t1(end) From c456e67bf2d06f54312ffdcd61090b1827b322f4 Mon Sep 17 00:00:00 2001 From: kid Date: Tue, 28 Jul 2026 17:27:02 +0800 Subject: [PATCH 2/4] fix: stop generate_series at terminal boundary --- .../functions-table/src/generate_series.rs | 49 ++++++++++++++++++- .../test_files/table_functions.slt | 12 +++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index 3f669fd94999d..668eee4454f13 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -275,6 +275,7 @@ impl GenerateSeriesTable { end: *end, step: *step, current: *start, + finished: false, batch_size, include_end: *include_end, name, @@ -315,6 +316,7 @@ impl GenerateSeriesTable { parsed_tz: Some(parsed_tz), tz_str: tz.clone(), }, + finished: false, batch_size, include_end: *include_end, name, @@ -344,6 +346,7 @@ impl GenerateSeriesTable { parsed_tz: None, tz_str: None, }, + finished: false, batch_size, include_end: *include_end, name, @@ -385,6 +388,7 @@ pub struct GenericSeriesState { step: T::StepType, batch_size: usize, current: T, + finished: bool, include_end: bool, name: &'static str, } @@ -425,6 +429,10 @@ impl LazyBatchGenerator for GenericSeriesState { } fn generate_next_batch(&mut self) -> Result> { + if self.finished { + return Ok(None); + } + let mut buf = Vec::with_capacity(self.batch_size); while buf.len() < self.batch_size @@ -437,11 +445,20 @@ impl LazyBatchGenerator for GenericSeriesState { .current .should_stop(self.end.clone(), &self.step, false) { - self.current.advance(&mut self.end, &self.step)?; + self.finished = true; break; } + let original_end = self.end.clone(); self.current.advance(&mut self.end, &self.step)?; + if self + .current + .should_stop(self.end.clone(), &self.step, self.include_end) + { + self.end = original_end; + self.finished = true; + break; + } } if buf.is_empty() { @@ -456,6 +473,7 @@ impl LazyBatchGenerator for GenericSeriesState { fn reset_state(&self) -> Arc> { let mut new = self.clone(); new.current = new.start.clone(); + new.finished = false; Arc::new(RwLock::new(new)) } } @@ -840,11 +858,40 @@ mod generate_series_tests { end: 5, step: 1, current: 1, + finished: false, + batch_size: 8192, + include_end: true, + name: "test", + }; + let batch = state.generate_next_batch()?.expect("missing batch"); + + let state_reset = state.reset_state(); + let reset_batch = state_reset + .write() + .generate_next_batch()? + .expect("missing reset batch"); + + assert_eq!(batch, reset_batch); + + Ok(()) + } + + #[test] + fn test_generic_series_state_reset_after_overflow() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let mut state = GenericSeriesState:: { + schema, + start: i64::MAX - 1, + end: i64::MAX, + step: 2, + current: i64::MAX - 1, + finished: false, batch_size: 8192, include_end: true, name: "test", }; let batch = state.generate_next_batch()?.expect("missing batch"); + assert!(state.generate_next_batch()?.is_none()); let state_reset = state.reset_state(); let reset_batch = state_reset diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.slt index c2c8ce24fa081..93d55a3edb12f 100644 --- a/datafusion/sqllogictest/test_files/table_functions.slt +++ b/datafusion/sqllogictest/test_files/table_functions.slt @@ -234,6 +234,18 @@ SELECT * FROM generate_series(DATE '0001-01-01', DATE '2000-01-01', INTERVAL '1' statement error DataFusion error: Error during planning: Second argument for generate_series is out of range of nanosecond timestamps SELECT * FROM generate_series(DATE '2000-01-01', DATE '3000-01-01', INTERVAL '1' DAY) +# Reaching the maximum representable date must not attempt to advance beyond it. +query P +SELECT * FROM generate_series(DATE '2262-04-11', DATE '2262-04-11', INTERVAL '1' DAY) +---- +2262-04-11T00:00:00 + +# Same for the maximum representable nanosecond timestamp. +query P +SELECT * FROM generate_series(TIMESTAMP '2262-04-11T23:47:16.854775807', TIMESTAMP '2262-04-11T23:47:16.854775807', INTERVAL '1' NANOSECOND) +---- +2262-04-11T23:47:16.854775807 + # UDF and UDTF `generate_series` can be used simultaneously query ? rowsort SELECT generate_series(1, t1.end) FROM generate_series(3, 5) as t1(end) From fc1c1feb50b0514c07df28ea7fd5e81e4145dbb7 Mon Sep 17 00:00:00 2001 From: kid Date: Thu, 30 Jul 2026 15:14:32 +0800 Subject: [PATCH 3/4] fix: preserve SeriesValue advance API --- .../functions-table/src/generate_series.rs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index 668eee4454f13..636f479286138 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -79,13 +79,17 @@ pub trait SeriesValue: fmt::Debug + Clone + Send + Sync + 'static { /// Check if we've reached the end of the series fn should_stop(&self, end: Self, step: &Self::StepType, include_end: bool) -> bool; - /// Advance to the next value in the series + /// Advance to the next value in the series. + fn advance(&mut self, step: &Self::StepType) -> Result<()>; + + /// Advance to the next value, adjusting the end of the series if needed. /// - /// If advancing would overflow the value range, `end` is updated so that - /// the series terminates after the current value (matching the behavior - /// of PostgreSQL and DuckDB, which return the reachable values instead of - /// erroring). - fn advance(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()>; + /// The default implementation preserves the behavior of [`Self::advance`]. + /// Implementations can override this method when they need to handle an + /// overflow by terminating the series after the current value. + fn advance_with_end(&mut self, _end: &mut Self, step: &Self::StepType) -> Result<()> { + self.advance(step) + } /// Create an Arrow array from a vector of values fn create_array(&self, values: Vec) -> Result; @@ -105,7 +109,12 @@ impl SeriesValue for i64 { reach_end_int64(*self, end, *step, include_end) } - fn advance(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()> { + fn advance(&mut self, step: &Self::StepType) -> Result<()> { + *self += step; + Ok(()) + } + + fn advance_with_end(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()> { if let Some(next) = self.checked_add(*step) { *self = next; } else { @@ -171,7 +180,7 @@ impl SeriesValue for TimestampValue { } } - fn advance(&mut self, _end: &mut Self, step: &Self::StepType) -> Result<()> { + fn advance(&mut self, step: &Self::StepType) -> Result<()> { let tz = self .parsed_tz .unwrap_or_else(|| Tz::from_str("+00:00").unwrap()); @@ -450,7 +459,7 @@ impl LazyBatchGenerator for GenericSeriesState { } let original_end = self.end.clone(); - self.current.advance(&mut self.end, &self.step)?; + self.current.advance_with_end(&mut self.end, &self.step)?; if self .current .should_stop(self.end.clone(), &self.step, self.include_end) From f949de123f79c62905dace3d363df8c1904a6968 Mon Sep 17 00:00:00 2001 From: kid Date: Thu, 30 Jul 2026 15:33:56 +0800 Subject: [PATCH 4/4] fix: stop timestamp series at range boundary --- .../functions-table/src/generate_series.rs | 21 +++++++++++++++++++ .../test_files/table_functions.slt | 13 ++++++++++++ 2 files changed, 34 insertions(+) diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index 636f479286138..f5e4df13899df 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -197,6 +197,27 @@ impl SeriesValue for TimestampValue { Ok(()) } + fn advance_with_end(&mut self, end: &mut Self, step: &Self::StepType) -> Result<()> { + let tz = self + .parsed_tz + .unwrap_or_else(|| Tz::from_str("+00:00").unwrap()); + if let Some(next_ts) = + TimestampNanosecondType::add_month_day_nano(self.value, *step, tz) + { + self.value = next_ts; + } else { + // Advancing would exceed the timestamp range. Clamp `end` so the + // series terminates after the current (last reachable) value. + let step_negative = step.months < 0 || step.days < 0 || step.nanoseconds < 0; + end.value = if step_negative { + self.value.saturating_add(1) + } else { + self.value.saturating_sub(1) + }; + } + Ok(()) + } + fn create_array(&self, values: Vec) -> Result { let array = TimestampNanosecondArray::from(values); diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.slt index 93d55a3edb12f..e67d898d71475 100644 --- a/datafusion/sqllogictest/test_files/table_functions.slt +++ b/datafusion/sqllogictest/test_files/table_functions.slt @@ -246,6 +246,19 @@ SELECT * FROM generate_series(TIMESTAMP '2262-04-11T23:47:16.854775807', TIMESTA ---- 2262-04-11T23:47:16.854775807 +# A timestamp step that exceeds the nanosecond range must terminate after the +# last reachable value instead of returning an overflow error. +query P +SELECT * FROM generate_series(TIMESTAMP '2262-04-11T23:47:16.854775806', TIMESTAMP '2262-04-11T23:47:16.854775807', INTERVAL '2' NANOSECOND) +---- +2262-04-11T23:47:16.854775806 + +# Same behavior for date series, which use the timestamp implementation. +query P +SELECT * FROM generate_series(DATE '2262-04-10', DATE '2262-04-11', INTERVAL '2' DAY) +---- +2262-04-10T00:00:00 + # UDF and UDTF `generate_series` can be used simultaneously query ? rowsort SELECT generate_series(1, t1.end) FROM generate_series(3, 5) as t1(end)