From ba4e15d17bff5f6ad2cd62d79fd62cd9e833bef5 Mon Sep 17 00:00:00 2001
From: tompng <tomoyapenguin@gmail.com>
Date: Tue, 10 Jun 2025 22:07:52 +0900
Subject: [PATCH] Fix edgecase segfault of BigDecimal#remainder

There is a possibility where coerced remainder returns Qnil.
NIL_P(BigDecimal_divremain(self, r, &d, &rv)) doesn't mean d and rv has a calculated value.
---
 ext/bigdecimal/bigdecimal.c        | 14 +++++++-------
 test/bigdecimal/test_bigdecimal.rb |  6 ++++++
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/ext/bigdecimal/bigdecimal.c b/ext/bigdecimal/bigdecimal.c
index 0f2bfc13..5798a5f8 100644
--- a/ext/bigdecimal/bigdecimal.c
+++ b/ext/bigdecimal/bigdecimal.c
@@ -2043,14 +2043,14 @@ BigDecimal_divremain(VALUE self, VALUE r, Real **dv, Real **rv)
 	b = GetVpValue(r, 0);
     }
 
-    if (!b) return DoSomeOne(self, r, rb_intern("remainder"));
+    if (!b) return Qfalse;
     SAVE(b);
 
     if (VpIsPosInf(b) || VpIsNegInf(b)) {
        GUARD_OBJ(*dv, NewZeroWrapLimited(1, 1));
        VpSetZero(*dv, 1);
        *rv = a;
-       return Qnil;
+       return Qtrue;
     }
 
     mx = (a->MaxPrec + b->MaxPrec) *VpBaseFig();
@@ -2074,7 +2074,7 @@ BigDecimal_divremain(VALUE self, VALUE r, Real **dv, Real **rv)
 
     *dv = d;
     *rv = ff;
-    return Qnil;
+    return Qtrue;
 }
 
 /* call-seq:
@@ -2087,11 +2087,11 @@ BigDecimal_divremain(VALUE self, VALUE r, Real **dv, Real **rv)
 static VALUE
 BigDecimal_remainder(VALUE self, VALUE r) /* remainder */
 {
-    VALUE  f;
     Real  *d, *rv = 0;
-    f = BigDecimal_divremain(self, r, &d, &rv);
-    if (!NIL_P(f)) return f;
-    return VpCheckGetValue(rv);
+    if (BigDecimal_divremain(self, r, &d, &rv)) {
+        return VpCheckGetValue(rv);
+    }
+    return DoSomeOne(self, r, rb_intern("remainder"));
 }
 
 /* call-seq:
diff --git a/test/bigdecimal/test_bigdecimal.rb b/test/bigdecimal/test_bigdecimal.rb
index 74dadd75..8f1474ad 100644
--- a/test/bigdecimal/test_bigdecimal.rb
+++ b/test/bigdecimal/test_bigdecimal.rb
@@ -1089,6 +1089,12 @@ def test_remainder_with_rational
     assert_kind_of(BigDecimal, BigDecimal("3").remainder(1.quo(3)))
   end
 
+  def test_remainder_coerce
+    o = Object.new
+    def o.coerce(x); [x, BigDecimal("3")]; end
+    assert_equal(BigDecimal("1.1"), BigDecimal("7.1").remainder(o))
+  end
+
   def test_divmod
     x = BigDecimal((2**100).to_s)
     assert_equal([(x / 3).floor, 1], x.divmod(3))