feat(kernel): mTLS client-certificate option + ABI-version check

KERNEL_REV

@@ -1 +1 @@
-9ac3f3d3f3e804d52d8890e890d9f8a8a617ec93
+760625bae82404b0195eaa4edeb8388cf3e9ab4d

connector.go

@@ -47,12 +47,12 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	} else {
 		// The experimental WithKernel* options have no Thrift-path equivalent — reject
 		// them loudly rather than silently ignore, so a caller who sets one (a
-		// trusted-CA bundle, a hostname-verify skip, a proxy, a retry budget, or a
-		// CloudFetch chunk cap) and forgets WithUseKernel learns the option had no
-		// effect instead of connecting as if it were never set. Every WithKernel*
-		// option allocates KernelExperimental, so this one gate covers them all; the
-		// message names the family rather than a stale subset that drifts as options
-		// are added.
+		// trusted-CA bundle, a hostname-verify skip, an mTLS client certificate, a
+		// proxy, a retry budget, or a CloudFetch chunk cap) and forgets WithUseKernel
+		// learns the option had no effect instead of connecting as if it were never
+		// set. Every WithKernel* option allocates KernelExperimental, so this one gate
+		// covers them all; the message names the family rather than a stale subset
+		// that drifts as options are added.
 		if c.cfg.KernelExperimental != nil {
 			return nil, fmt.Errorf("databricks: a WithKernel* option %w; "+
 				"add WithUseKernel(true) or remove it", dbsqlerr.ErrRequiresKernelBackend)
@@ -615,6 +615,40 @@ func WithKernelSkipHostnameVerify() ConnOption {
 	}
 }
 
+// WithKernelClientCertificate configures a client certificate + private key for
+// mutual TLS (mTLS) on the kernel backend. cert is a PEM leaf certificate,
+// optionally followed by its intermediate chain; key is the matching PEM private
+// key (use PKCS#8 for portability). Both halves are required together — they map
+// to the single paired kernel_session_config_set_tls_client_certificate, so
+// cert-without-key is unrepresentable. The private key is never logged.
+//
+// EXPERIMENTAL, kernel-only: the default (Thrift) backend rejects this at connect.
+func WithKernelClientCertificate(cert, key []byte) ConnOption {
+	return func(c *config.Config) {
+		// Copy defensively (matching KernelExperimentalConfig.DeepCopy) so a
+		// caller mutating the buffers between NewConnector and Connect can't change
+		// the identity out from under us.
+		k := kernelExperimental(c)
+		// Record that mTLS was explicitly requested, independent of whether the
+		// caller passed non-empty bytes. Without this marker an empty cert+key
+		// pair (e.g. from a failed PEM load) is indistinguishable from the option
+		// never being called, and validateKernelConfig would accept it while
+		// applyKernelTLS silently skipped the setter — connecting with no client
+		// identity. The marker lets validation reject the incomplete request loudly.
+		k.TLSClientCertConfigured = true
+		if len(cert) > 0 {
+			k.TLSClientCertPEM = append([]byte(nil), cert...)
+		} else {
+			k.TLSClientCertPEM = cert
+		}
+		if len(key) > 0 {
+			k.TLSClientKeyPEM = append([]byte(nil), key...)
+		} else {
+			k.TLSClientKeyPEM = key
+		}
+	}
+}
+
 // KernelProxy is the explicit-proxy configuration for WithKernelProxy. Its fields
 // are named so a call site can't transpose the credentials — the four values are
 // all strings, so a positional signature would let a Username/Password swap (or a

doc.go

@@ -241,6 +241,9 @@ WithKernel* prefix marks them experimental):
     not read SSL_CERT_FILE.
   - WithKernelSkipHostnameVerify() skips only the hostname check while keeping chain
     validation (finer-grained than WithSkipTLSHostVerify).
+  - WithKernelClientCertificate(cert, key) configures a client certificate + private
+    key for mutual TLS (mTLS). Both PEM halves are required together; the key is
+    never logged.
   - WithKernelProxy(KernelProxy{URL, Username, Password, BypassHosts}) sets an explicit
     HTTP proxy, overriding the HTTP(S)_PROXY / NO_PROXY environment the kernel path
     otherwise mirrors. The fields are named (not positional) so the four same-typed

internal/backend/kernel/abi.go

@@ -0,0 +1,23 @@
+package kernel
+
+import "fmt"
+
+// compareABI reports whether the C-ABI version compiled into the linked kernel
+// library (got) matches the version the driver's header declares (want): it
+// returns a descriptive, operator-facing error on mismatch and nil when they
+// agree.
+//
+// It is split out from checkABIVersion (in the cgo-tagged cgo.go) as pure,
+// cgo-free logic so the mismatch verdict and its message are unit-testable under
+// the default CGO_ENABLED=0 build. The negative path can't otherwise be reached
+// from a test: a real build always links a .a and header produced together, and
+// abiVersions() reads fixed cgo constants with no injection seam. checkABIVersion
+// runs abiVersions() through this once per process (sync.Once-cached), on the
+// first connect.
+func compareABI(got, want uint32) error {
+	if got != want {
+		return fmt.Errorf("databricks: kernel ABI version mismatch: linked library reports %d, "+
+			"driver header expects %d; rebuild the kernel static lib and header together (make kernel-lib)", got, want)
+	}
+	return nil
+}

internal/backend/kernel/abi_test.go

@@ -0,0 +1,37 @@
+package kernel
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestCompareABI covers both verdicts of the ABI-version handshake without the
+// cgo symbols, so it runs under the default CGO_ENABLED=0 build. The matching
+// (happy) path is additionally proven against the real linked library by
+// TestABIVersionMatches in the cgo build; the mismatch path — the runtime hazard
+// the check exists for (a driver header linked against a differently-built
+// prebuilt .a) — is otherwise unreachable from a test, since a real build always
+// links a matching .a + header.
+func TestCompareABI(t *testing.T) {
+	// Matching versions must produce no error: checkABIVersion opens the session.
+	for _, v := range []uint32{0, 1, 42} {
+		if err := compareABI(v, v); err != nil {
+			t.Errorf("compareABI(%d, %d) = %v, want nil for matching versions", v, v, err)
+		}
+	}
+
+	// Mismatched versions must produce an error so checkABIVersion refuses to open
+	// (rather than silently misread status codes / error-struct fields). The
+	// message must name both versions and the remediation so an operator hitting a
+	// stale prebuilt .a knows what to rebuild.
+	err := compareABI(2, 1)
+	if err == nil {
+		t.Fatal("compareABI(2, 1) = nil, want an error for mismatched versions")
+	}
+	msg := err.Error()
+	for _, want := range []string{"ABI version mismatch", "reports 2", "expects 1", "make kernel-lib"} {
+		if !strings.Contains(msg, want) {
+			t.Errorf("mismatch error %q does not contain %q", msg, want)
+		}
+	}
+}

internal/backend/kernel/backend.go

@@ -79,14 +79,18 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error {
 	// Deferred (tracked): this ctx is only checked here, at entry — once inside the
 	// blocking kernel_session_open there is no way to honor a deadline/cancel that
 	// fires mid-connect (a slow warehouse cold-start or a connect-time network
-	// partition blocks until the kernel returns on its own). Same class and same
-	// root cause as the CloseSession no-deadline note below (the kernel C ABI
-	// exposes no deadline/cancellation on the session-lifecycle calls); the fix is
-	// the same kernel-side change (a deadline arg or cancel handle), so it's grouped
-	// with that follow-up rather than fixed Go-side with a watchdog here.
+	// partition blocks until the kernel returns on its own). The fix is a kernel-side
+	// change (a deadline arg or cancel handle); tracked as a follow-up.
 	if err := ctx.Err(); err != nil {
 		return err
 	}
+	// Verify the linked kernel library's C-ABI version matches the header the
+	// driver compiled against before making any other kernel call — a mismatch
+	// means every status code / error struct we read afterward could be
+	// misinterpreted. Runs once per process; cheap and cached thereafter.
+	if err := checkABIVersion(); err != nil {
+		return err
+	}
 	initKernelLogging()
 	klogCtx(ctx, "OpenSession host=%s httpPath=%s warehouse=%s", k.cfg.Host, k.cfg.HTTPPath, k.cfg.WarehouseID)
 
@@ -233,10 +237,9 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error {
 }
 
 // applyKernelTLS forwards the experimental kernel-only TLS knobs to the session
-// config: a custom CA bundle and an independent hostname-skip. Each is a no-op
-// when its field is unset, so this is safe to call unconditionally. (mTLS client
-// cert/key is a separate follow-up — it needs a kernel C-ABI setter that is not
-// yet on kernel main.)
+// config: a custom CA bundle, an independent hostname-skip, and an mTLS client
+// certificate + key. Each is a no-op when its field is unset, so this is safe to
+// call unconditionally.
 func (k *KernelBackend) applyKernelTLS(cfg *C.KernelSessionConfig) error {
 	if len(k.cfg.TLSTrustedCertsPEM) > 0 {
 		ca := newCBytes(k.cfg.TLSTrustedCertsPEM)
@@ -254,6 +257,20 @@ func (k *KernelBackend) applyKernelTLS(cfg *C.KernelSessionConfig) error {
 			return fmt.Errorf("kernel: set_tls_skip_hostname_verification: %w", toConnError(err))
 		}
 	}
+	// mTLS client identity. The cert and key travel as a pair (WithKernelClientCertificate
+	// sets both or neither), forwarded via the single paired setter; checking the
+	// cert is enough. The key bytes go to owned Rust memory and are never logged.
+	if len(k.cfg.TLSClientCertPEM) > 0 {
+		cert := newCBytes(k.cfg.TLSClientCertPEM)
+		defer cert.free()
+		key := newCBytes(k.cfg.TLSClientKeyPEM)
+		defer key.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_tls_client_certificate(cfg, cert.ptr, cert.len, key.ptr, key.len)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_tls_client_certificate: %w", toConnError(err))
+		}
+	}
 	return nil
 }
 
@@ -463,14 +480,15 @@ func (k *KernelBackend) runNamespaceStmt(ctx context.Context, sql string) error
 	return closeErr
 }
 
-// CloseSession tears down the server-side session. Best-effort: the kernel's
-// close is async (see the C header), so an error is logged, not hard-failed.
+// CloseSession tears down the server-side session. Best-effort: kernel_session
+// _close initiates the delete without waiting (see the C header), so it does not
+// block and an error is logged, not hard-failed.
 //
 // Deferred (tracked): this ignores ctx and blocks in the synchronous call() until
 // kernel_session_close returns, with no deadline — a stalled kernel-side close
 // (e.g. a shutdown-time network partition) can block database/sql pool cleanup.
-// A bounded close needs either a kernel_session_close_blocking with a deadline or
-// a Go-side watchdog; grouped with the kernel C-ABI follow-ups.
+// A bounded close needs a kernel C-ABI deadline/cancel handle; tracked as a
+// follow-up.
 func (k *KernelBackend) CloseSession(ctx context.Context) error {
 	if k.session == nil {
 		return nil

internal/backend/kernel/cgo.go

@@ -153,6 +153,43 @@ func initKernelLogging() {
 	})
 }
 
+// abiCheckOnce ensures the ABI-version handshake runs at most once per process
+// (the linked library can't change under us mid-run), and abiCheckErr caches its
+// outcome so every OpenSession after the first returns the same verdict cheaply.
+var (
+	abiCheckOnce sync.Once
+	abiCheckErr  error
+)
+
+// checkABIVersion verifies the C-ABI version compiled into the linked kernel
+// library matches the version the driver's header declares. The build-time
+// status-code assertions in this file catch header-vs-source drift at compile
+// time; this catches the runtime hazard those can't see — a driver built against
+// one header linked against a differently-built prebuilt .a (the eventual
+// download-a-prebuilt-lib distribution path). A mismatch means the
+// KernelStatusCode enum / KernelError struct layout the driver reads may not
+// match what the library writes, so we refuse to open rather than silently
+// misread status codes / error fields. Runs once; the result is cached. The
+// pure got-vs-want verdict lives in compareABI (untagged) so its mismatch path
+// is testable without the cgo symbols.
+func checkABIVersion() error {
+	abiCheckOnce.Do(func() {
+		got, want := abiVersions()
+		klog("checkABIVersion got=%d want=%d", got, want)
+		abiCheckErr = compareABI(got, want)
+	})
+	return abiCheckErr
+}
+
+// abiVersions returns (library version, header version): the C-ABI version
+// compiled into the linked kernel library and the version the driver's header
+// declares. A production-side seam so tests can assert the handshake without
+// putting cgo in a _test.go file (which Go forbids); not used in production
+// beyond checkABIVersion.
+func abiVersions() (got, want uint32) {
+	return uint32(C.kernel_abi_version()), uint32(C.DATABRICKS_KERNEL_ABI_VERSION)
+}
+
 // call runs a fallible kernel entry point and, on a non-Success status, reads
 // the kernel's thread-local last error into a Go error.
 //

internal/backend/kernel/config.go

@@ -35,6 +35,14 @@ type Config struct {
 	TLSTrustedCertsPEM    []byte
 	TLSSkipHostnameVerify bool
 
+	// TLSClientCertPEM / TLSClientKeyPEM are the mTLS client cert (+ optional
+	// chain) and matching private key (from WithKernelClientCertificate). They
+	// travel as a pair — both set or both empty — and are forwarded via the single
+	// paired kernel_session_config_set_tls_client_certificate. The key is never
+	// logged.
+	TLSClientCertPEM []byte
+	TLSClientKeyPEM  []byte
+
 	// ProxyURL configures an HTTP proxy. It is either resolved for this endpoint
 	// from the same HTTP(S)_PROXY / NO_PROXY environment the Thrift path uses
 	// (NO_PROXY applied during resolution), or set explicitly via WithKernelProxy.

internal/backend/kernel/kernel_test.go

@@ -59,13 +59,20 @@ func TestSetAuthByMode(t *testing.T) {
 // drifted.
 func TestSetKernelTLS(t *testing.T) {
 	ca := []byte("-----BEGIN CERTIFICATE-----\nca\n-----END CERTIFICATE-----\n")
+	cert := []byte("-----BEGIN CERTIFICATE-----\nleaf\n-----END CERTIFICATE-----\n")
+	// Assemble the key's PEM marker at runtime so the repo's secret scanner
+	// doesn't flag a literal BEGIN-PRIVATE-KEY (the bytes are opaque to the
+	// marshalling path under test).
+	key := []byte("-----BEGIN " + "PRIVATE" + " KEY-----\nkey\n-----END PRIVATE KEY-----\n")
 	cases := []struct {
 		name string
 		cfg  Config
 	}{
 		{"trusted certs only", Config{TLSTrustedCertsPEM: ca}},
 		{"skip hostname only", Config{TLSSkipHostnameVerify: true}},
 		{"both together", Config{TLSTrustedCertsPEM: ca, TLSSkipHostnameVerify: true}},
+		{"client certificate (mTLS)", Config{TLSClientCertPEM: cert, TLSClientKeyPEM: key}},
+		{"all together", Config{TLSTrustedCertsPEM: ca, TLSSkipHostnameVerify: true, TLSClientCertPEM: cert, TLSClientKeyPEM: key}},
 		{"none (no-op)", Config{}},
 	}
 	for _, c := range cases {
@@ -77,6 +84,23 @@ func TestSetKernelTLS(t *testing.T) {
 	}
 }
 
+// TestABIVersionMatches asserts the linked kernel library's C-ABI version
+// matches the header the driver compiled against — the same handshake
+// checkABIVersion runs at connect. It exercises the real cgo symbols
+// (kernel_abi_version + the DATABRICKS_KERNEL_ABI_VERSION macro), so a stale
+// .a-vs-header pairing fails here at test time rather than misreading status
+// codes at runtime. It also asserts checkABIVersion() returns nil for the
+// matched pair the test binary links.
+func TestABIVersionMatches(t *testing.T) {
+	got, want := abiVersions()
+	if got != want {
+		t.Fatalf("kernel_abi_version() = %d, header DATABRICKS_KERNEL_ABI_VERSION = %d", got, want)
+	}
+	if err := checkABIVersion(); err != nil {
+		t.Errorf("checkABIVersion() = %v, want nil for the linked (matching) library", err)
+	}
+}
+
 // TestSetProxy exercises the real kernel_session_config_set_proxy cgo setter via
 // the trySetProxy seam across every field combination — proving the arg
 // marshalling and the NULL-for-empty handling of the optional username / password
@@ -89,7 +113,7 @@ func TestSetProxy(t *testing.T) {
 	}{
 		{"url only", Config{ProxyURL: "http://proxy:3128"}},
 		{"url + credentials", Config{ProxyURL: "http://proxy:3128", ProxyUsername: "u", ProxyPassword: "p"}},
-		{"url + bypass", Config{ProxyURL: "http://proxy:3128", ProxyBypassHosts: "localhost,*.internal"}}, //nolint:gosec // G101: test literals, not real credentials
+		{"url + bypass", Config{ProxyURL: "http://proxy:3128", ProxyBypassHosts: "localhost,*.internal"}},                                       //nolint:gosec // G101: test literals, not real credentials
 		{"all fields", Config{ProxyURL: "http://proxy:3128", ProxyUsername: "u", ProxyPassword: "p", ProxyBypassHosts: "localhost,*.internal"}}, //nolint:gosec // G101: test literals, not real credentials
 		{"none (no-op)", Config{}},
 	}

internal/config/config.go

@@ -76,6 +76,23 @@ type KernelExperimentalConfig struct {
 	// of the blanket InsecureSkipVerify (which relaxes both chain and hostname).
 	// Maps to kernel_session_config_set_tls_skip_hostname_verification.
 	TLSSkipHostnameVerify bool
+	// TLSClientCertPEM / TLSClientKeyPEM are the client leaf cert (+ optional
+	// chain) and matching private key for mutual TLS (mTLS). They travel as a
+	// pair — both set or both nil (WithKernelClientCertificate is the only setter
+	// and takes both) — and map to the single paired
+	// kernel_session_config_set_tls_client_certificate. The key is never logged.
+	TLSClientCertPEM []byte
+	TLSClientKeyPEM  []byte
+	// TLSClientCertConfigured records that WithKernelClientCertificate was
+	// invoked, independent of whether the caller passed non-empty bytes. It
+	// exists because an empty cert+key pair is otherwise indistinguishable from
+	// the option never being called (KernelExperimental can be non-nil from any
+	// other WithKernel* option), which would let a caller who explicitly asked
+	// for mTLS — e.g. after a failed/empty PEM load — silently connect with no
+	// client identity (applyKernelTLS gates the setter on a non-empty cert).
+	// validateKernelConfig uses this marker to reject an incomplete mTLS request
+	// loudly instead of failing open. Never forwarded to the kernel C ABI.
+	TLSClientCertConfigured bool
 
 	// ProxyURL / ProxyUsername / ProxyPassword / ProxyBypassHosts configure an
 	// explicit HTTP proxy on the kernel path (WithKernelProxy), overriding the
@@ -117,17 +134,24 @@ func (k *KernelExperimentalConfig) DeepCopy() *KernelExperimentalConfig {
 		return nil
 	}
 	cp := &KernelExperimentalConfig{
-		TLSSkipHostnameVerify: k.TLSSkipHostnameVerify,
-		ProxyURL:              k.ProxyURL,
-		ProxyUsername:         k.ProxyUsername,
-		ProxyPassword:         k.ProxyPassword,
-		ProxyBypassHosts:      k.ProxyBypassHosts,
-		RetryOverallTimeout:   k.RetryOverallTimeout,
-		MaxChunksInMemory:     k.MaxChunksInMemory,
+		TLSSkipHostnameVerify:   k.TLSSkipHostnameVerify,
+		TLSClientCertConfigured: k.TLSClientCertConfigured,
+		ProxyURL:                k.ProxyURL,
+		ProxyUsername:           k.ProxyUsername,
+		ProxyPassword:           k.ProxyPassword,
+		ProxyBypassHosts:        k.ProxyBypassHosts,
+		RetryOverallTimeout:     k.RetryOverallTimeout,
+		MaxChunksInMemory:       k.MaxChunksInMemory,
 	}
 	if k.TLSTrustedCertsPEM != nil {
 		cp.TLSTrustedCertsPEM = append([]byte(nil), k.TLSTrustedCertsPEM...)
 	}
+	if k.TLSClientCertPEM != nil {
+		cp.TLSClientCertPEM = append([]byte(nil), k.TLSClientCertPEM...)
+	}
+	if k.TLSClientKeyPEM != nil {
+		cp.TLSClientKeyPEM = append([]byte(nil), k.TLSClientKeyPEM...)
+	}
 	return cp
 }