dynamic_partition_overwrite after an in-transaction update_spec loses rows #4192

import tempfile
import pyarrow as pa
from pyiceberg.catalog.sql import SqlCatalog
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.transforms import IdentityTransform
from pyiceberg.types import LongType, NestedField

cat = SqlCatalog("c", uri="sqlite:///:memory:", warehouse=f"file://{tempfile.mkdtemp()}")
cat.create_namespace("ns")
schema = Schema(NestedField(1, "a", LongType()), NestedField(2, "b", LongType()))
t = cat.create_table("ns.t", schema=schema, partition_spec=PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "a")))
df = lambda *rs: pa.Table.from_pylist([{"a": a, "b": b} for a, b in rs], schema=schema.as_arrow())
t.append(df((1, 9), (2, 1)))

with t.transaction() as tx:
    with tx.update_spec() as u:
        u.remove_field("a")
        u.add_identity("b")
    tx.dynamic_partition_overwrite(df((1, 2)))

rows = lambda f="True": sorted((r["a"], r["b"]) for r in t.scan(row_filter=f).to_arrow().to_pylist())
print(rows())
# [(1, 2), (1, 9)], should be [(1, 2), (1, 9), (2, 1)]
print(rows("b == 2"))
# [], should be [(1, 2)]
# the new file is recorded under b = 1, its a value