From c12e49ccb6b06ac749beb41f1d6950be1f02cdaa Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 14 Sep 2026 15:29:19 +0700 Subject: [PATCH] feat(connections): edit a SQLite database on an SSH server in place, run on the server --- CHANGELOG.md | 6 + .../SQLiteAgentBackend.swift | 550 ++++++++++++++++++ .../SQLiteAgentProtocol.swift | 401 +++++++++++++ .../SQLiteExecutionBackend.swift | 401 +++++++++++++ Plugins/SQLiteDriverPlugin/SQLitePlugin.swift | 502 +--------------- TablePro/AppDelegate.swift | 1 + .../AI/Chat/Tools/ExecuteQueryChatTool.swift | 18 +- .../DatabaseManager+RemoteSQLite.swift | 83 +++ .../Core/Database/DatabaseManager+SSH.swift | 5 + .../DatabaseManager+SystemEvents.swift | 2 + .../Database/DatabaseManager+Tunnel.swift | 1 + .../DatabaseManager+Verification.swift | 4 + .../Plugins/PluginManager+Registration.swift | 6 + ...uginMetadataRegistry+CuratedDefaults.swift | 3 +- .../Core/Plugins/PluginMetadataRegistry.swift | 7 + .../RemoteSQLite/RemoteSQLiteAdmission.swift | 48 ++ .../RemoteSQLiteAgentSource.swift | 481 +++++++++++++++ .../RemoteSQLiteExecChannelOpener.swift | 113 ++++ .../RemoteSQLiteTransportManager.swift | 158 +++++ .../RemoteSQLite/RemoteSQLiteTunnel.swift | 293 ++++++++++ .../Core/RemoteSQLite/RemoteSQLiteWire.swift | 34 ++ .../ConnectionStageLabelFormatter.swift | 2 +- .../Connection/ConnectionTunnelKind.swift | 36 +- .../Connection/SSHTunnelFormState.swift | 12 +- TablePro/Models/Connection/SSHTypes.swift | 17 + .../ConnectionFormCoordinator+SafeMode.swift | 2 +- .../ConnectionFormCoordinator+Transport.swift | 7 + .../Panes/NetworkPaneView.swift | 2 +- .../Transports/SSHTransportSections.swift | 28 +- .../RemoteSQLiteAdmissionTests.swift | 53 ++ .../RemoteSQLiteAgentPipeTests.swift | 180 ++++++ .../RemoteSQLite/RemoteSQLiteModelTests.swift | 92 +++ .../RemoteSQLiteWireParityTests.swift | 30 + .../SQLiteAgentProtocolTests.swift | 97 +++ docs/connections/remote-database-files.mdx | 43 +- docs/databases/sqlite.mdx | 4 +- project.yml | 1 + 37 files changed, 3216 insertions(+), 507 deletions(-) create mode 100644 Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift create mode 100644 Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift create mode 100644 Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift create mode 100644 TablePro/Core/Database/DatabaseManager+RemoteSQLite.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteAdmission.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift create mode 100644 TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift create mode 100644 TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift create mode 100644 TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift create mode 100644 TableProTests/RemoteSQLite/RemoteSQLiteModelTests.swift create mode 100644 TableProTests/RemoteSQLite/RemoteSQLiteWireParityTests.swift create mode 100644 TableProTests/RemoteSQLite/SQLiteAgentProtocolTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c6ecc630..186399f1bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - What's New window, from Help > What's New. - Update install mode and check frequency in the anonymous usage heartbeat. - OceanBase MySQL-mode connection type on the MySQL driver. (#1748) +- On the Server mode for a SQLite Remote Database File, editing a database on an SSH server in place with statements run on the server. (#2831) ### Changed @@ -44,6 +45,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Row selected by Back or Forward dropped once the table finished loading. - Data grid jumping to the top on a later reload after a page change failed. +### Security + +- SQLite denies the `fts3_tokenizer` function, which could crash the app from a crafted query on any connection. +- The AI assistant refuses statements that read or write files or run server-side code (ATTACH, LOAD, VACUUM INTO), matching the MCP server. (#2831) + ## [0.74.0] - 2026-09-13 ### Added diff --git a/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift b/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift new file mode 100644 index 0000000000..4bd14f7376 --- /dev/null +++ b/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift @@ -0,0 +1,550 @@ +// +// SQLiteAgentBackend.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +/// Runs a SQLite driver's statements on an SSH server through the Python agent, over a loopback +/// socket the app's transport hands to a fresh exec channel per connection. +/// +/// The backend speaks the framed protocol in `SQLiteAgentProtocol`. It owns one socket, a reader +/// thread that assembles reply frames, and a heartbeat that keeps the agent's watchdog from +/// reaping the session. The database, every value and every statement travel as protocol fields; +/// the exec command line the server runs carries none of them. +actor SQLiteAgentBackend: SQLiteExecutionBackend { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteAgentBackend") + + private let connection: SQLiteAgentConnection + private let path: String + private let token: String + private var busyTimeoutMilliseconds: Int32 = 0 + + nonisolated let canceller: SQLiteCanceller + nonisolated var resolvedServerVersion: String? { connection.cachedServerVersion } + + init(host: String, port: Int, path: String, token: String) { + let connection = SQLiteAgentConnection(host: host, port: port) + self.connection = connection + self.path = path + self.token = token + self.canceller = SQLiteRemoteCanceller(connection: connection) + } + + func open() async throws { + try connection.connect(token: token) + let ready = try await connection.hello( + path: path, + busyTimeoutMilliseconds: UInt32(max(0, busyTimeoutMilliseconds)) + ) + connection.cachedServerVersion = ready.sqliteVersion + Self.logger.info( + "Remote SQLite agent ready: SQLite \(ready.sqliteVersion, privacy: .public), Python \(ready.pythonVersion, privacy: .public)" + ) + } + + func close() { + connection.close() + } + + nonisolated func abortConnect() { + connection.close() + } + + func applyBusyTimeout(_ milliseconds: Int32) { + busyTimeoutMilliseconds = milliseconds + connection.setBusyTimeout(UInt32(max(0, milliseconds))) + } + + func executeQuery(_ query: String) async throws -> SQLiteRawResult { + try await run(query, parameters: []) + } + + func executeParameterizedQuery(_ query: String, parameters: [PluginCellValue]) async throws -> SQLiteRawResult { + try await run(query, parameters: parameters.map(Self.encode)) + } + + private func run(_ query: String, parameters: [SQLiteAgentValue]) async throws -> SQLiteRawResult { + let startTime = Date() + let accumulator = SQLiteAgentResultAccumulator() + + let outcome = try await withTaskCancellationHandler { + try await connection.execute( + sql: query, + parameters: parameters, + // The buffered path holds the whole result in memory, so it takes the same emergency + // ceiling the local backend applies; the streaming path below stays uncapped. + rowCap: UInt32(clamping: PluginRowLimits.emergencyMax), + onHeader: { header in accumulator.setHeader(header) }, + onRows: { columnCount, values in + guard columnCount > 0 else { return } + var rows: [[PluginCellValue]] = [] + var index = 0 + while index < values.count { + rows.append(values[index..<(index + columnCount)].map(Self.decode)) + index += columnCount + } + accumulator.appendRows(rows) + } + ) + } onCancel: { + connection.sendCancel() + } + + let (columns, columnTypeNames, rows) = accumulator.snapshot() + return SQLiteRawResult( + columns: columns, + columnTypeNames: columnTypeNames, + rows: rows, + rowsAffected: columns.isEmpty ? Int(outcome.changes) : 0, + executionTime: Date().timeIntervalSince(startTime), + isTruncated: outcome.truncated + ) + } + + func streamQuery( + _ query: String, + continuation: AsyncThrowingStream.Continuation + ) async throws { + _ = try await withTaskCancellationHandler { + try await connection.execute( + sql: query, + parameters: [], + rowCap: 0, + onHeader: { header in + continuation.yield(.header(PluginStreamHeader( + columns: header.map(\.name), + columnTypeNames: header.map { $0.declaredType ?? "" }, + estimatedRowCount: nil + ))) + }, + onRows: { columnCount, values in + guard columnCount > 0 else { return } + var batch: [PluginRow] = [] + var index = 0 + while index < values.count { + batch.append(values[index..<(index + columnCount)].map(Self.decode)) + index += columnCount + } + continuation.yield(.rows(batch)) + } + ) + } onCancel: { + connection.sendCancel() + } + continuation.finish() + } + + private static func encode(_ value: PluginCellValue) -> SQLiteAgentValue { + switch value { + case .null: return .null + case .text(let string): return .text(Data(string.utf8)) + case .bytes(let data): return .blob(data) + } + } + + /// The agent renders integers and reals as their SQLite text form, exactly as the local backend + /// reads them through `sqlite3_column_text`, so a value reads the same whichever backend served + /// it. Only NULL and BLOB are distinct types on the wire. + private static func decode(_ value: SQLiteAgentValue) -> PluginCellValue { + switch value { + case .null: return .null + // swiftlint:disable:next optional_data_string_conversion + case .text(let data): return .text(String(decoding: data, as: UTF8.self)) + case .blob(let data): return .bytes(data) + } + } +} + +/// Collects a buffered result as the reader thread delivers header and row frames. The frames for +/// one statement arrive in order on that one thread, and the awaiting caller reads the snapshot only +/// after the statement's done frame has resumed it; the lock is there so the `@Sendable` callbacks +/// are safe by construction rather than by that argument alone. +final class SQLiteAgentResultAccumulator: @unchecked Sendable { + private let lock = NSLock() + private var columns: [String] = [] + private var columnTypeNames: [String] = [] + private var rows: [[PluginCellValue]] = [] + + func setHeader(_ header: [SQLiteAgentColumn]) { + lock.lock() + defer { lock.unlock() } + columns = header.map(\.name) + columnTypeNames = header.map { $0.declaredType ?? "" } + } + + func appendRows(_ newRows: [[PluginCellValue]]) { + lock.lock() + defer { lock.unlock() } + rows.append(contentsOf: newRows) + } + + func snapshot() -> (columns: [String], columnTypeNames: [String], rows: [[PluginCellValue]]) { + lock.lock() + defer { lock.unlock() } + return (columns, columnTypeNames, rows) + } +} + +/// Writes a cancel frame to the agent from the Stop button, without waiting on the backend actor. +final class SQLiteRemoteCanceller: SQLiteCanceller, @unchecked Sendable { + private let connection: SQLiteAgentConnection + + init(connection: SQLiteAgentConnection) { + self.connection = connection + } + + func cancel() { + connection.sendCancel() + } +} + +struct SQLiteAgentReady: Sendable { + let sqliteVersion: String + let pythonVersion: String +} + +struct SQLiteAgentExecuteOutcome: Sendable { + let changes: Int64 + let truncated: Bool +} + +/// The socket, the reader thread, and the framed protocol conversation with one agent. +/// +/// Reads block on a dedicated thread that assembles frames and routes them to the one operation in +/// flight; the backend actor runs at most one at a time. Writes are serialized under a lock, so +/// the reader thread's cancel and heartbeat frames never interleave with a partial request. +final class SQLiteAgentConnection: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteAgentConnection") + private static let readyTimeout: TimeInterval = 30 + private static let heartbeatInterval: TimeInterval = 15 + + private let host: String + private let port: Int + + private let stateLock = NSLock() + private var socketFD: Int32 = -1 + private var pending: SQLiteAgentOperation? + private var terminalFailure: Error? + private var closed = false + + private let writeLock = NSLock() + private var reader = SQLiteAgentFrameReader() + private var readerThread: Thread? + private var heartbeatTimer: DispatchSourceTimer? + + private let versionLock = NSLock() + private var _cachedServerVersion: String? + var cachedServerVersion: String? { + get { versionLock.lock(); defer { versionLock.unlock() }; return _cachedServerVersion } + set { versionLock.lock(); _cachedServerVersion = newValue; versionLock.unlock() } + } + + init(host: String, port: Int) { + self.host = host + self.port = port + } + + func connect(token: String) throws { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw SQLitePluginError.connectionFailed("could not create a local socket") } + + var addr = sockaddr_in() + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = in_port_t(port).bigEndian + addr.sin_addr.s_addr = inet_addr(host) + + let connectResult = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard connectResult == 0 else { + Darwin.close(fd) + throw SQLitePluginError.connectionFailed("could not reach the remote SQLite session") + } + + stateLock.lock() + socketFD = fd + closed = false + stateLock.unlock() + + try writeAll(SQLiteAgentProtocol.admissionPreamble(token: token)) + + let thread = Thread { [weak self] in self?.readLoop() } + thread.name = "com.TablePro.sqlite.agent-reader" + thread.stackSize = 512 * 1_024 + readerThread = thread + thread.start() + } + + func hello(path: String, busyTimeoutMilliseconds: UInt32) async throws -> SQLiteAgentReady { + let request = SQLiteAgentRequest.hello( + protocolVersion: SQLiteAgentProtocol.version, + path: path, + busyTimeoutMilliseconds: busyTimeoutMilliseconds + ) + let ready: SQLiteAgentReady = try await withOperation(deadline: Self.readyTimeout) { _ in + try self.writeAll(SQLiteAgentFrameEncoder.encode(request)) + } + startHeartbeat() + return ready + } + + func execute( + sql: String, + parameters: [SQLiteAgentValue], + rowCap: UInt32, + onHeader: @escaping (@Sendable ([SQLiteAgentColumn]) -> Void), + onRows: @escaping (@Sendable (Int, [SQLiteAgentValue]) -> Void) + ) async throws -> SQLiteAgentExecuteOutcome { + let request = SQLiteAgentRequest.execute(sql: sql, parameters: parameters, rowCap: rowCap) + return try await withOperation(deadline: nil) { operation in + operation.onHeader = onHeader + operation.onRows = onRows + try self.writeAll(SQLiteAgentFrameEncoder.encode(request)) + } + } + + func setBusyTimeout(_ milliseconds: UInt32) { + try? writeAll(SQLiteAgentFrameEncoder.encode(.setBusyTimeout(milliseconds: milliseconds))) + } + + func sendCancel() { + try? writeAll(SQLiteAgentFrameEncoder.encode(.cancel)) + } + + func close() { + stateLock.lock() + guard !closed else { stateLock.unlock(); return } + closed = true + let fd = socketFD + socketFD = -1 + let inflight = pending + pending = nil + stateLock.unlock() + + heartbeatTimer?.cancel() + heartbeatTimer = nil + if fd >= 0 { + shutdown(fd, SHUT_RDWR) + Darwin.close(fd) + } + inflight?.fail(SQLitePluginError.notConnected) + } + + // MARK: - Operation lifecycle + + private func withOperation( + deadline: TimeInterval?, + _ send: @escaping (SQLiteAgentOperation) throws -> Void + ) async throws -> T { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let operation = SQLiteAgentOperation { result in + switch result { + case .success(let value): + if let typed = value as? T { + continuation.resume(returning: typed) + } else { + continuation.resume(throwing: SQLitePluginError.queryFailed("unexpected reply from the remote SQLite session")) + } + case .failure(let error): + continuation.resume(throwing: error) + } + } + + stateLock.lock() + if closed { + stateLock.unlock() + continuation.resume(throwing: SQLitePluginError.notConnected) + return + } + /// A terminal reply that arrived before this operation registered, such as the launcher's + /// "no python3" notice landing before the hello op was set, is delivered to this + /// operation rather than lost. The reader thread stores it under the same lock, so the + /// two orderings, store-then-register and register-then-route, both reach here correctly. + if let stored = terminalFailure { + terminalFailure = nil + stateLock.unlock() + operation.fail(stored) + return + } + pending = operation + stateLock.unlock() + + if let deadline { + DispatchQueue.global().asyncAfter(deadline: .now() + deadline) { [weak self] in + self?.failIfPending(operation, error: SQLitePluginError.connectionFailed("the remote SQLite session did not answer")) + } + } + + do { + try send(operation) + } catch { + failIfPending(operation, error: error) + } + } + } + + private func failIfPending(_ operation: SQLiteAgentOperation, error: Error) { + stateLock.lock() + guard pending === operation else { stateLock.unlock(); return } + pending = nil + stateLock.unlock() + operation.fail(error) + } + + private func completePending(_ complete: (SQLiteAgentOperation) -> Void) { + stateLock.lock() + let operation = pending + pending = nil + stateLock.unlock() + guard let operation else { return } + complete(operation) + } + + /// Delivers a terminal failure to the pending operation, or stores the first one for the next + /// operation to register when none is in flight yet. This is what keeps the launcher's + /// "no python3" notice from being lost when it arrives before the hello op is set. + private func completeOrStoreFailure(_ error: Error) { + stateLock.lock() + if let operation = pending { + pending = nil + stateLock.unlock() + operation.fail(error) + return + } + if terminalFailure == nil { terminalFailure = error } + stateLock.unlock() + } + + // MARK: - Reader + + private func readLoop() { + var buffer = [UInt8](repeating: 0, count: 32 * 1_024) + while true { + stateLock.lock() + let fd = socketFD + let isClosed = closed + stateLock.unlock() + guard !isClosed, fd >= 0 else { break } + + let count = read(fd, &buffer, buffer.count) + if count <= 0 { break } + reader.append(Data(buffer[0..= 0 else { throw SQLitePluginError.notConnected } + + try data.withUnsafeBytes { raw in + guard let base = raw.baseAddress else { return } + var offset = 0 + while offset < raw.count { + let written = write(fd, base.advanced(by: offset), raw.count - offset) + if written <= 0 { throw SQLitePluginError.notConnected } + offset += written + } + } + } +} + +/// One request/response exchange with the agent. The reader thread drives its callbacks and +/// completes it exactly once, whichever of a done frame, an error frame, a timeout or a socket +/// close arrives first. +final class SQLiteAgentOperation: @unchecked Sendable { + private let completion: (Result) -> Void + private let lock = NSLock() + private var finished = false + + var onHeader: (@Sendable ([SQLiteAgentColumn]) -> Void)? + var onRows: (@Sendable (Int, [SQLiteAgentValue]) -> Void)? + + init(completion: @escaping (Result) -> Void) { + self.completion = completion + } + + func succeed(_ value: Any) { + guard claim() else { return } + completion(.success(value)) + } + + func fail(_ error: Error) { + guard claim() else { return } + completion(.failure(error)) + } + + private func claim() -> Bool { + lock.lock() + defer { lock.unlock() } + guard !finished else { return false } + finished = true + return true + } +} diff --git a/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift b/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift new file mode 100644 index 0000000000..f4917247b2 --- /dev/null +++ b/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift @@ -0,0 +1,401 @@ +// +// SQLiteAgentProtocol.swift +// TablePro +// + +import Foundation + +enum SQLiteAgentProtocol { + static let version: UInt32 = 1 + static let maxFrameLength = 64 * 1_024 * 1_024 + static let backendFieldKey = "sqliteBackend" + static let agentBackendValue = "agent" + static let tokenFieldKey = "sqliteAgentToken" + static let admissionPrefix = "TPRSQL1 " + static let launcherNoticePrefix = "TPRSQL:" + static let noPythonNotice = "TPRSQL:NO_PYTHON" + + static func admissionPreamble(token: String) -> Data { + Data((admissionPrefix + token + "\n").utf8) + } +} + +enum SQLiteAgentValue: Equatable, Sendable { + case null + case text(Data) + case blob(Data) +} + +struct SQLiteAgentColumn: Equatable, Sendable { + let name: String + let declaredType: String? +} + +struct SQLiteAgentFailure: Equatable, Sendable { + static let unsupportedProtocol: UInt32 = 1 + static let pythonTooOld: UInt32 = 2 + static let libraryUnavailable: UInt32 = 3 + static let openFailed: UInt32 = 4 + static let callbacksUnavailable: UInt32 = 5 + static let malformedRequest: UInt32 = 6 + + let code: UInt32 + let message: String +} + +enum SQLiteAgentRequest: Equatable, Sendable { + case hello(protocolVersion: UInt32, path: String, busyTimeoutMilliseconds: UInt32) + case execute(sql: String, parameters: [SQLiteAgentValue], rowCap: UInt32) + case cancel + case heartbeat + case setBusyTimeout(milliseconds: UInt32) +} + +enum SQLiteAgentReply: Equatable, Sendable { + case ready(protocolVersion: UInt32, sqliteVersion: String, pythonVersion: String) + case failure(SQLiteAgentFailure) + case header([SQLiteAgentColumn]) + case rows(columnCount: Int, values: [SQLiteAgentValue]) + case done(changes: Int64, truncated: Bool) + case error(code: Int32, message: String) + case launcherNotice(String) +} + +enum SQLiteAgentProtocolError: Error, Equatable { + case frameTooLarge(Int) + case emptyFrame + case truncatedPayload + case trailingBytes + case unknownOpcode(UInt8) + case unknownValueTag(UInt8) + case rowShapeMismatch +} + +enum SQLiteAgentOpcode { + static let hello: UInt8 = 0x01 + static let execute: UInt8 = 0x02 + static let cancel: UInt8 = 0x03 + static let heartbeat: UInt8 = 0x04 + static let setBusyTimeout: UInt8 = 0x05 + static let ready: UInt8 = 0x81 + static let failure: UInt8 = 0x82 + static let header: UInt8 = 0x83 + static let rows: UInt8 = 0x84 + static let done: UInt8 = 0x85 + static let error: UInt8 = 0x86 +} + +enum SQLiteAgentFrameEncoder { + static func encode(_ request: SQLiteAgentRequest) -> Data { + var payload = SQLiteAgentPayloadWriter() + let opcode: UInt8 + switch request { + case .hello(let protocolVersion, let path, let busyTimeout): + opcode = SQLiteAgentOpcode.hello + payload.appendUInt32(protocolVersion) + payload.appendString(path) + payload.appendUInt32(busyTimeout) + case .execute(let sql, let parameters, let rowCap): + opcode = SQLiteAgentOpcode.execute + payload.appendString(sql) + payload.appendUInt32(UInt32(parameters.count)) + parameters.forEach { payload.appendValue($0) } + payload.appendUInt32(rowCap) + case .cancel: + opcode = SQLiteAgentOpcode.cancel + case .heartbeat: + opcode = SQLiteAgentOpcode.heartbeat + case .setBusyTimeout(let milliseconds): + opcode = SQLiteAgentOpcode.setBusyTimeout + payload.appendUInt32(milliseconds) + } + return frame(opcode: opcode, payload: payload.bytes) + } + + static func encode(_ reply: SQLiteAgentReply) -> Data { + var payload = SQLiteAgentPayloadWriter() + let opcode: UInt8 + switch reply { + case .ready(let protocolVersion, let sqliteVersion, let pythonVersion): + opcode = SQLiteAgentOpcode.ready + payload.appendUInt32(protocolVersion) + payload.appendString(sqliteVersion) + payload.appendString(pythonVersion) + case .failure(let failure): + opcode = SQLiteAgentOpcode.failure + payload.appendUInt32(failure.code) + payload.appendString(failure.message) + case .header(let columns): + opcode = SQLiteAgentOpcode.header + payload.appendUInt32(UInt32(columns.count)) + for column in columns { + payload.appendString(column.name) + if let declaredType = column.declaredType { + payload.appendUInt8(1) + payload.appendString(declaredType) + } else { + payload.appendUInt8(0) + } + } + case .rows(let columnCount, let values): + opcode = SQLiteAgentOpcode.rows + let rowCount = columnCount == 0 ? 0 : values.count / columnCount + payload.appendUInt32(UInt32(columnCount)) + payload.appendUInt32(UInt32(rowCount)) + values.forEach { payload.appendValue($0) } + case .done(let changes, let truncated): + opcode = SQLiteAgentOpcode.done + payload.appendInt64(changes) + payload.appendUInt8(truncated ? 1 : 0) + case .error(let code, let message): + opcode = SQLiteAgentOpcode.error + payload.appendUInt32(UInt32(bitPattern: code)) + payload.appendString(message) + case .launcherNotice(let notice): + return Data((notice + "\n").utf8) + } + return frame(opcode: opcode, payload: payload.bytes) + } + + private static func frame(opcode: UInt8, payload: [UInt8]) -> Data { + var writer = SQLiteAgentPayloadWriter() + writer.appendUInt32(UInt32(payload.count + 1)) + writer.appendUInt8(opcode) + writer.bytes.append(contentsOf: payload) + return Data(writer.bytes) + } +} + +struct SQLiteAgentFrameReader { + private var buffer: [UInt8] = [] + private var readIndex = 0 + private var expectsLauncherNotice = true + + mutating func append(_ data: Data) { + if readIndex > 0, readIndex == buffer.count { + buffer.removeAll(keepingCapacity: true) + readIndex = 0 + } + buffer.append(contentsOf: data) + } + + mutating func nextReply() throws -> SQLiteAgentReply? { + if expectsLauncherNotice, let notice = try launcherNotice() { + return notice + } + guard let (opcode, payload) = try nextBody() else { return nil } + var cursor = SQLiteAgentPayloadCursor(bytes: payload) + let reply = try decodeReply(opcode: opcode, cursor: &cursor) + guard cursor.isAtEnd else { throw SQLiteAgentProtocolError.trailingBytes } + return reply + } + + mutating func nextRequest() throws -> SQLiteAgentRequest? { + expectsLauncherNotice = false + guard let (opcode, payload) = try nextBody() else { return nil } + var cursor = SQLiteAgentPayloadCursor(bytes: payload) + let request = try decodeRequest(opcode: opcode, cursor: &cursor) + guard cursor.isAtEnd else { throw SQLiteAgentProtocolError.trailingBytes } + return request + } + + private mutating func launcherNotice() throws -> SQLiteAgentReply? { + let prefix = Array(SQLiteAgentProtocol.launcherNoticePrefix.utf8) + let available = buffer.count - readIndex + let comparedCount = min(available, prefix.count) + guard buffer[readIndex..<(readIndex + comparedCount)].elementsEqual(prefix[0..= prefix.count else { return nil } + guard let newline = buffer[readIndex...].firstIndex(of: 0x0A) else { + guard available <= 4_096 else { throw SQLiteAgentProtocolError.frameTooLarge(available) } + return nil + } + let notice = String(decoding: buffer[readIndex.. (UInt8, ArraySlice)? { + let available = buffer.count - readIndex + guard available >= 4 else { return nil } + var lengthCursor = SQLiteAgentPayloadCursor(bytes: buffer[readIndex..<(readIndex + 4)]) + let length = Int(try lengthCursor.readUInt32()) + guard length > 0 else { throw SQLiteAgentProtocolError.emptyFrame } + guard length <= SQLiteAgentProtocol.maxFrameLength else { + throw SQLiteAgentProtocolError.frameTooLarge(length) + } + guard available >= 4 + length else { return nil } + expectsLauncherNotice = false + let bodyStart = readIndex + 4 + let opcode = buffer[bodyStart] + let payload = buffer[(bodyStart + 1)..<(bodyStart + length)] + readIndex = bodyStart + length + return (opcode, payload) + } + + private func decodeReply(opcode: UInt8, cursor: inout SQLiteAgentPayloadCursor) throws -> SQLiteAgentReply { + switch opcode { + case SQLiteAgentOpcode.ready: + return .ready( + protocolVersion: try cursor.readUInt32(), + sqliteVersion: try cursor.readString(), + pythonVersion: try cursor.readString() + ) + case SQLiteAgentOpcode.failure: + return .failure(SQLiteAgentFailure(code: try cursor.readUInt32(), message: try cursor.readString())) + case SQLiteAgentOpcode.header: + let count = Int(try cursor.readUInt32()) + var columns: [SQLiteAgentColumn] = [] + columns.reserveCapacity(min(count, 4_096)) + for _ in 0.. SQLiteAgentRequest { + switch opcode { + case SQLiteAgentOpcode.hello: + return .hello( + protocolVersion: try cursor.readUInt32(), + path: try cursor.readString(), + busyTimeoutMilliseconds: try cursor.readUInt32() + ) + case SQLiteAgentOpcode.execute: + let sql = try cursor.readString() + let count = Int(try cursor.readUInt32()) + guard count <= cursor.remaining else { throw SQLiteAgentProtocolError.truncatedPayload } + var parameters: [SQLiteAgentValue] = [] + parameters.reserveCapacity(count) + for _ in 0..(_ data: Bytes) where Bytes.Element == UInt8 { + appendUInt32(UInt32(data.count)) + bytes.append(contentsOf: data) + } + + mutating func appendString(_ string: String) { + appendBytes(Array(string.utf8)) + } + + mutating func appendValue(_ value: SQLiteAgentValue) { + switch value { + case .null: + appendUInt8(0) + case .text(let data): + appendUInt8(1) + appendBytes(data) + case .blob(let data): + appendUInt8(2) + appendBytes(data) + } + } +} + +private struct SQLiteAgentPayloadCursor { + let bytes: ArraySlice + private var offset: Int + + init(bytes: ArraySlice) { + self.bytes = bytes + self.offset = bytes.startIndex + } + + var isAtEnd: Bool { offset == bytes.endIndex } + var remaining: Int { bytes.endIndex - offset } + + mutating func readUInt8() throws -> UInt8 { + guard remaining >= 1 else { throw SQLiteAgentProtocolError.truncatedPayload } + defer { offset += 1 } + return bytes[offset] + } + + mutating func readUInt32() throws -> UInt32 { + guard remaining >= 4 else { throw SQLiteAgentProtocolError.truncatedPayload } + defer { offset += 4 } + return bytes[offset..<(offset + 4)].reduce(UInt32(0)) { ($0 << 8) | UInt32($1) } + } + + mutating func readInt64() throws -> Int64 { + guard remaining >= 8 else { throw SQLiteAgentProtocolError.truncatedPayload } + defer { offset += 8 } + let raw = bytes[offset..<(offset + 8)].reduce(UInt64(0)) { ($0 << 8) | UInt64($1) } + return Int64(bitPattern: raw) + } + + mutating func readBytes() throws -> Data { + let count = Int(try readUInt32()) + guard remaining >= count else { throw SQLiteAgentProtocolError.truncatedPayload } + defer { offset += count } + return Data(bytes[offset..<(offset + count)]) + } + + mutating func readString() throws -> String { + String(decoding: try readBytes(), as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + } + + mutating func readValue() throws -> SQLiteAgentValue { + let tag = try readUInt8() + switch tag { + case 0: + return .null + case 1: + return .text(try readBytes()) + case 2: + return .blob(try readBytes()) + default: + throw SQLiteAgentProtocolError.unknownValueTag(tag) + } + } +} diff --git a/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift b/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift new file mode 100644 index 0000000000..5a72f3e7e3 --- /dev/null +++ b/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift @@ -0,0 +1,401 @@ +// +// SQLiteExecutionBackend.swift +// TablePro +// + +import Foundation +import os +import SQLite3 +import TableProPluginKit + +/// Where a SQLite driver's statements actually run. +/// +/// A file-backed connection opens the database with the app's own `libsqlite3` (the local backend). +/// A connection whose database lives on an SSH server runs its statements on the server through a +/// small agent, because SQLite's file locking is only reliable for a process on the same host as +/// the file. Both answer the same driver, so every schema, metadata and write path is written once +/// against this protocol and neither knows which backend it is talking to. +protocol SQLiteExecutionBackend: Actor { + /// The SQLite version that will run the statements: the app's library for a local file, the + /// server's for a remote one. Cached at connect time; nil until then. + nonisolated var resolvedServerVersion: String? { get } + + /// Ends a running statement, and a wait on a locked database, without waiting on the actor. A + /// query holds the actor for its whole life, so cancel cannot be an actor-isolated method. + nonisolated var canceller: SQLiteCanceller { get } + + func open() async throws + func close() async + + /// Stops an in-flight `open()` without waiting on the actor, for a connect the user cancelled. + /// `Task.cancel()` is cooperative and cannot interrupt a blocking wait, so the remote backend + /// tears down its socket here; a local open has nothing to abort. + nonisolated func abortConnect() + + func applyBusyTimeout(_ milliseconds: Int32) async + func executeQuery(_ query: String) async throws -> SQLiteRawResult + func executeParameterizedQuery(_ query: String, parameters: [PluginCellValue]) async throws -> SQLiteRawResult + func streamQuery( + _ query: String, + continuation: AsyncThrowingStream.Continuation + ) async throws +} + +/// The one thing about a running query that must be reachable while the query holds the backend +/// actor: the request to stop it. The local backend interrupts the SQLite handle; the remote +/// backend writes a cancel frame to the agent. +protocol SQLiteCanceller: Sendable { + func cancel() +} + +extension SQLiteExecutionBackend { + nonisolated func abortConnect() {} +} + +struct SQLiteRawResult: Sendable { + let columns: [String] + let columnTypeNames: [String] + let rows: [[PluginCellValue]] + let rowsAffected: Int + let executionTime: TimeInterval + let isTruncated: Bool +} + +// MARK: - Authorizer + +/// Denies the SQLite functions that turn plain SQL into a native-code primitive on the machine +/// running the query. `fts3_tokenizer` registers an arbitrary pointer as a tokenizer from a bound +/// blob and dereferences it (measured: a SIGSEGV on the app's own libsqlite3 and on every server +/// build tried), and `load_extension` loads a shared library. Both are denied on every SQLite +/// connection, local or remote, so a crafted statement from the editor, an import, or an AI or MCP +/// client cannot reach them. +enum SQLiteAuthorizer { + private static let function: Int32 = 31 + private static let denied: Set = ["fts3_tokenizer", "load_extension"] + + private static let callback: @convention(c) ( + UnsafeMutableRawPointer?, Int32, + UnsafePointer?, UnsafePointer?, UnsafePointer?, UnsafePointer? + ) -> Int32 = { _, action, _, arg2, _, _ in + guard action == function, let arg2 else { return SQLITE_OK } + let name = String(cString: arg2).lowercased() + return denied.contains(name) ? SQLITE_DENY : SQLITE_OK + } + + static func install(on db: OpaquePointer?) { + sqlite3_set_authorizer(db, callback, nil) + } +} + +// MARK: - Busy Wait + +/// Ends a wait on a locked database when the user presses Stop, and after the configured timeout. +/// +/// `sqlite3_busy_timeout` cannot do the first of those: it sleeps inside SQLite with nothing to +/// interrupt it, and `sqlite3_interrupt` does not reach a connection that is waiting for a lock +/// rather than running a statement. Measured against SQLite 3.54.0 with a second connection +/// holding `BEGIN EXCLUSIVE`: the interrupt was ignored and the waiter ran the full 60 seconds +/// before returning `SQLITE_BUSY`. A busy handler is the documented way to keep that decision, +/// because it is called back on every retry and stops the wait by returning zero. +/// +/// Read and written from whichever thread is stepping a statement and from the caller of Stop, so +/// every access takes the lock. +final class SQLiteBusyState: @unchecked Sendable { + private let lock = NSLock() + private var isCancelled = false + private var timeoutMilliseconds: Int32 = 0 + + /// How long one retry waits. Also the granularity at which Stop is noticed. + static let retryIntervalMilliseconds: Int32 = 10 + + func setTimeout(milliseconds: Int32) { + lock.lock() + defer { lock.unlock() } + timeoutMilliseconds = milliseconds + } + + func beginOperation() { + lock.lock() + defer { lock.unlock() } + isCancelled = false + } + + func cancel() { + lock.lock() + defer { lock.unlock() } + isCancelled = true + } + + /// - Parameter retryCount: How many times SQLite has already called back for this lock. + /// - Returns: `true` to wait and retry, `false` to give up and let the step return `SQLITE_BUSY`. + func shouldRetry(afterRetryCount retryCount: Int32) -> Bool { + lock.lock() + defer { lock.unlock() } + guard !isCancelled else { return false } + guard timeoutMilliseconds > 0 else { return true } + return retryCount * Self.retryIntervalMilliseconds < timeoutMilliseconds + } +} + +let sqliteBusyHandler: @convention(c) (UnsafeMutableRawPointer?, Int32) -> Int32 = { context, retryCount in + guard let context else { return 0 } + let state = Unmanaged.fromOpaque(context).takeUnretainedValue() + guard state.shouldRetry(afterRetryCount: retryCount) else { return 0 } + usleep(UInt32(SQLiteBusyState.retryIntervalMilliseconds) * 1_000) + return 1 +} + +// MARK: - Local Backend + +/// Interrupts the local SQLite handle from the Stop button. Holds the handle behind a lock because +/// the interrupting thread and the actor stepping the statement reach it at once. +final class SQLiteLocalCanceller: SQLiteCanceller, @unchecked Sendable { + private let lock = NSLock() + private let busyState: SQLiteBusyState + private var handle: OpaquePointer? + + init(busyState: SQLiteBusyState) { + self.busyState = busyState + } + + func setHandle(_ handle: OpaquePointer?) { + lock.lock() + self.handle = handle + lock.unlock() + } + + func cancel() { + busyState.cancel() + lock.lock() + let handle = self.handle + lock.unlock() + guard let handle else { return } + sqlite3_interrupt(handle) + } +} + +/// Opens a SQLite database with the app's own `libsqlite3` and runs statements against it. +actor SQLiteLocalBackend: SQLiteExecutionBackend { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteLocalBackend") + + private var db: OpaquePointer? + private let busyState: SQLiteBusyState + private let path: String + private let localCanceller: SQLiteLocalCanceller + + nonisolated let canceller: SQLiteCanceller + nonisolated var resolvedServerVersion: String? { String(cString: sqlite3_libversion()) } + + init(path: String) { + self.path = path + let busyState = SQLiteBusyState() + self.busyState = busyState + let canceller = SQLiteLocalCanceller(busyState: busyState) + self.localCanceller = canceller + self.canceller = canceller + } + + var isConnected: Bool { db != nil } + + func open() throws { + let expandedPath = expandPath(path) + if !FileManager.default.fileExists(atPath: expandedPath) { + let directory = (expandedPath as NSString).deletingLastPathComponent + try? FileManager.default.createDirectory(atPath: directory, withIntermediateDirectories: true) + } + + let result = sqlite3_open(expandedPath, &db) + if result != SQLITE_OK { + let errorMessage = db.map { String(cString: sqlite3_errmsg($0)) } ?? "Unknown SQLite error" + throw SQLitePluginError.connectionFailed(errorMessage) + } + SQLiteAuthorizer.install(on: db) + installBusyHandler() + localCanceller.setHandle(db) + } + + func close() { + localCanceller.setHandle(nil) + if db != nil { + sqlite3_close(db) + db = nil + } + } + + func applyBusyTimeout(_ milliseconds: Int32) { + busyState.setTimeout(milliseconds: milliseconds) + } + + private func installBusyHandler() { + guard let db else { return } + sqlite3_busy_handler(db, sqliteBusyHandler, Unmanaged.passUnretained(busyState).toOpaque()) + } + + private func expandPath(_ path: String) -> String { + path.hasPrefix("~") ? NSString(string: path).expandingTildeInPath : path + } + + func executeQuery(_ query: String) throws -> SQLiteRawResult { + try runStatement(query, parameters: nil) + } + + func executeParameterizedQuery(_ query: String, parameters: [PluginCellValue]) throws -> SQLiteRawResult { + try runStatement(query, parameters: parameters) + } + + private func runStatement(_ query: String, parameters: [PluginCellValue]?) throws -> SQLiteRawResult { + guard let db else { throw SQLitePluginError.notConnected } + busyState.beginOperation() + + let startTime = Date() + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, query, -1, &statement, nil) == SQLITE_OK else { + throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + defer { sqlite3_finalize(statement) } + + if let parameters { + try bind(parameters, to: statement, db: db) + } + + let columnCount = sqlite3_column_count(statement) + let (columns, columnTypeNames) = Self.columnMetadata(statement, count: columnCount) + + var rows: [[PluginCellValue]] = [] + var truncated = false + var stepResult = sqlite3_step(statement) + while stepResult == SQLITE_ROW { + if rows.count >= PluginRowLimits.emergencyMax { + truncated = true + break + } + rows.append(Self.readRow(statement, columnCount: columnCount)) + stepResult = sqlite3_step(statement) + } + + if !truncated, stepResult != SQLITE_DONE { + throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + + let rowsAffected = columns.isEmpty ? Int(sqlite3_changes(db)) : 0 + return SQLiteRawResult( + columns: columns, + columnTypeNames: columnTypeNames, + rows: rows, + rowsAffected: rowsAffected, + executionTime: Date().timeIntervalSince(startTime), + isTruncated: truncated + ) + } + + func streamQuery( + _ query: String, + continuation: AsyncThrowingStream.Continuation + ) throws { + busyState.beginOperation() + guard let db else { throw SQLitePluginError.notConnected } + + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, query, -1, &statement, nil) == SQLITE_OK else { + throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + + let columnCount = sqlite3_column_count(statement) + let (columns, columnTypeNames) = Self.columnMetadata(statement, count: columnCount) + continuation.yield(.header(PluginStreamHeader( + columns: columns, columnTypeNames: columnTypeNames, estimatedRowCount: nil + ))) + + let batchSize = 5_000 + var batch: [PluginRow] = [] + batch.reserveCapacity(batchSize) + + var stepResult = sqlite3_step(statement) + while stepResult == SQLITE_ROW { + if Task.isCancelled { + if !batch.isEmpty { continuation.yield(.rows(batch)) } + sqlite3_finalize(statement) + continuation.finish(throwing: CancellationError()) + return + } + batch.append(Self.readRow(statement, columnCount: columnCount)) + if batch.count >= batchSize { + continuation.yield(.rows(batch)) + batch.removeAll(keepingCapacity: true) + } + stepResult = sqlite3_step(statement) + } + + if !batch.isEmpty { continuation.yield(.rows(batch)) } + + guard stepResult == SQLITE_DONE else { + let message = String(cString: sqlite3_errmsg(db)) + sqlite3_finalize(statement) + continuation.finish(throwing: SQLitePluginError.queryFailed(message)) + return + } + sqlite3_finalize(statement) + continuation.finish() + } + + private func bind(_ parameters: [PluginCellValue], to statement: OpaquePointer?, db: OpaquePointer) throws { + let sqliteTransient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + for (index, param) in parameters.enumerated() { + let bindIndex = Int32(index + 1) + let bindResult: Int32 + switch param { + case .null: + bindResult = sqlite3_bind_null(statement, bindIndex) + case .text(let stringValue): + bindResult = sqlite3_bind_text(statement, bindIndex, stringValue, -1, sqliteTransient) + case .bytes(let data): + bindResult = data.withUnsafeBytes { rawBuffer -> Int32 in + sqlite3_bind_blob(statement, bindIndex, rawBuffer.baseAddress, Int32(data.count), sqliteTransient) + } + } + if bindResult != SQLITE_OK { + throw SQLitePluginError.queryFailed("Failed to bind parameter \(index): \(String(cString: sqlite3_errmsg(db)))") + } + } + } + + private static func columnMetadata(_ statement: OpaquePointer?, count: Int32) -> ([String], [String]) { + var columns: [String] = [] + var columnTypeNames: [String] = [] + for i in 0.. [PluginCellValue] { + var row: [PluginCellValue] = [] + for i in 0.. 0, let blobPtr = sqlite3_column_blob(statement, i) { + row.append(.bytes(Data(bytes: blobPtr, count: byteCount))) + } else { + row.append(.bytes(Data())) + } + } else if let text = sqlite3_column_text(statement, i) { + row.append(.text(String(cString: text))) + } else { + row.append(.null) + } + } + return row + } +} diff --git a/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index fc5ae1ff65..2611836473 100644 --- a/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -114,445 +114,16 @@ final class SQLitePlugin: NSObject, TableProPlugin, DriverPlugin { } } -// MARK: - Busy Wait - -/// Ends a wait on a locked database when the user presses Stop, and after the configured timeout. -/// -/// `sqlite3_busy_timeout` cannot do the first of those: it sleeps inside SQLite with nothing to -/// interrupt it, and `sqlite3_interrupt` does not reach a connection that is waiting for a lock -/// rather than running a statement. Measured against SQLite 3.54.0 with a second connection -/// holding `BEGIN EXCLUSIVE`: the interrupt was ignored and the waiter ran the full 60 seconds -/// before returning `SQLITE_BUSY`. A busy handler is the documented way to keep that decision, -/// because it is called back on every retry and stops the wait by returning zero. -/// -/// Read and written from whichever thread is stepping a statement and from the caller of Stop, so -/// every access takes the lock. -private final class SQLiteBusyState: @unchecked Sendable { - private let lock = NSLock() - private var isCancelled = false - private var timeoutMilliseconds: Int32 = 0 - - /// How long one retry waits. Also the granularity at which Stop is noticed. - static let retryIntervalMilliseconds: Int32 = 10 - - func setTimeout(milliseconds: Int32) { - lock.lock() - defer { lock.unlock() } - timeoutMilliseconds = milliseconds - } - - func beginOperation() { - lock.lock() - defer { lock.unlock() } - isCancelled = false - } - - func cancel() { - lock.lock() - defer { lock.unlock() } - isCancelled = true - } - - /// - Parameter retryCount: How many times SQLite has already called back for this lock. - /// - Returns: `true` to wait and retry, `false` to give up and let the step return `SQLITE_BUSY`. - func shouldRetry(afterRetryCount retryCount: Int32) -> Bool { - lock.lock() - defer { lock.unlock() } - guard !isCancelled else { return false } - guard timeoutMilliseconds > 0 else { return true } - return retryCount * Self.retryIntervalMilliseconds < timeoutMilliseconds - } -} - -private let sqliteBusyHandler: @convention(c) (UnsafeMutableRawPointer?, Int32) -> Int32 = { context, retryCount in - guard let context else { return 0 } - let state = Unmanaged.fromOpaque(context).takeUnretainedValue() - guard state.shouldRetry(afterRetryCount: retryCount) else { return 0 } - usleep(UInt32(SQLiteBusyState.retryIntervalMilliseconds) * 1_000) - return 1 -} - -// MARK: - SQLite Connection Actor - -private actor SQLiteConnectionActor { - private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteConnectionActor") - - private var db: OpaquePointer? - private let busyState: SQLiteBusyState - - init(busyState: SQLiteBusyState) { - self.busyState = busyState - } - - var isConnected: Bool { db != nil } - - func open(path: String) throws { - let result = sqlite3_open(path, &db) - - if result != SQLITE_OK { - let errorMessage = db.map { String(cString: sqlite3_errmsg($0)) } - ?? "Unknown SQLite error" - throw SQLitePluginError.connectionFailed(errorMessage) - } - installBusyHandler() - } - - func close() { - if db != nil { - sqlite3_close(db) - db = nil - } - } - - func applyBusyTimeout(_ milliseconds: Int32) { - busyState.setTimeout(milliseconds: milliseconds) - } - - func beginBusyOperation() { - busyState.beginOperation() - } - - private func installBusyHandler() { - guard let db else { return } - sqlite3_busy_handler(db, sqliteBusyHandler, Unmanaged.passUnretained(busyState).toOpaque()) - } - - var dbHandleForInterrupt: Int { db.map { Int(bitPattern: $0) } ?? 0 } - - func executeQuery(_ query: String) throws -> SQLiteRawResult { - guard let db else { - throw SQLitePluginError.notConnected - } - busyState.beginOperation() - - let startTime = Date() - var statement: OpaquePointer? - - let prepareResult = sqlite3_prepare_v2(db, query, -1, &statement, nil) - - if prepareResult != SQLITE_OK { - let errorMessage = String(cString: sqlite3_errmsg(db)) - throw SQLitePluginError.queryFailed(errorMessage) - } - - defer { - sqlite3_finalize(statement) - } - - let columnCount = sqlite3_column_count(statement) - var columns: [String] = [] - var columnTypeNames: [String] = [] - - for i in 0..= PluginRowLimits.emergencyMax { - truncated = true - break - } - - var row: [PluginCellValue] = [] - - for i in 0.. 0, let blobPtr = sqlite3_column_blob(statement, i) { - row.append(.bytes(Data(bytes: blobPtr, count: byteCount))) - } else { - row.append(.bytes(Data())) - } - } else if let text = sqlite3_column_text(statement, i) { - row.append(.text(String(cString: text))) - } else { - row.append(.null) - } - } - - rows.append(row) - stepResult = sqlite3_step(statement) - } - - if !truncated, stepResult != SQLITE_DONE { - throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db))) - } - - if columns.isEmpty { - rowsAffected = Int(sqlite3_changes(db)) - } - - let executionTime = Date().timeIntervalSince(startTime) - - return SQLiteRawResult( - columns: columns, - columnTypeNames: columnTypeNames, - rows: rows, - rowsAffected: rowsAffected, - executionTime: executionTime, - isTruncated: truncated - ) - } - - func streamQuery(_ query: String, continuation: AsyncThrowingStream.Continuation) throws { - busyState.beginOperation() - guard let db else { - throw SQLitePluginError.notConnected - } - - var statement: OpaquePointer? - - let prepareResult = sqlite3_prepare_v2(db, query, -1, &statement, nil) - if prepareResult != SQLITE_OK { - let errorMessage = String(cString: sqlite3_errmsg(db)) - throw SQLitePluginError.queryFailed(errorMessage) - } - - let columnCount = sqlite3_column_count(statement) - var columns: [String] = [] - var columnTypeNames: [String] = [] - - for i in 0.. 0, let blobPtr = sqlite3_column_blob(statement, i) { - row.append(.bytes(Data(bytes: blobPtr, count: byteCount))) - } else { - row.append(.bytes(Data())) - } - } else if let text = sqlite3_column_text(statement, i) { - row.append(.text(String(cString: text))) - } else { - row.append(.null) - } - } - - batch.append(row) - if batch.count >= batchSize { - continuation.yield(.rows(batch)) - batch.removeAll(keepingCapacity: true) - } - stepResult = sqlite3_step(statement) - } - - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } - - // A step that ends on anything but `SQLITE_DONE` stopped early: a locked database, an I/O - // error on the volume, a corrupt page. Finishing the stream normally would report the rows - // read so far as the whole table, which is indistinguishable from an empty one. - guard stepResult == SQLITE_DONE else { - let message = String(cString: sqlite3_errmsg(db)) - sqlite3_finalize(statement) - continuation.finish(throwing: SQLitePluginError.queryFailed(message)) - return - } - - sqlite3_finalize(statement) - continuation.finish() - } - - func executeParameterizedQuery(_ query: String, parameters: [PluginCellValue]) throws -> SQLiteRawResult { - guard let db else { - throw SQLitePluginError.notConnected - } - busyState.beginOperation() - - let startTime = Date() - var statement: OpaquePointer? - - let prepareResult = sqlite3_prepare_v2(db, query, -1, &statement, nil) - - if prepareResult != SQLITE_OK { - let errorMessage = String(cString: sqlite3_errmsg(db)) - throw SQLitePluginError.queryFailed(errorMessage) - } - - defer { - sqlite3_finalize(statement) - } - - let sqliteTransient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) - - for (index, param) in parameters.enumerated() { - let bindIndex = Int32(index + 1) - let bindResult: Int32 - - switch param { - case .null: - bindResult = sqlite3_bind_null(statement, bindIndex) - case .text(let stringValue): - bindResult = sqlite3_bind_text(statement, bindIndex, stringValue, -1, sqliteTransient) - case .bytes(let data): - bindResult = data.withUnsafeBytes { rawBuffer -> Int32 in - let baseAddress = rawBuffer.baseAddress - return sqlite3_bind_blob(statement, bindIndex, baseAddress, Int32(data.count), sqliteTransient) - } - } - - if bindResult != SQLITE_OK { - let errorMessage = String(cString: sqlite3_errmsg(db)) - throw SQLitePluginError.queryFailed( - "Failed to bind parameter \(index): \(errorMessage)" - ) - } - } - - let columnCount = sqlite3_column_count(statement) - var columns: [String] = [] - var columnTypeNames: [String] = [] - - for i in 0..= PluginRowLimits.emergencyMax { - truncated = true - break - } - - var row: [PluginCellValue] = [] - - for i in 0.. 0, let blobPtr = sqlite3_column_blob(statement, i) { - row.append(.bytes(Data(bytes: blobPtr, count: byteCount))) - } else { - row.append(.bytes(Data())) - } - } else if let text = sqlite3_column_text(statement, i) { - row.append(.text(String(cString: text))) - } else { - row.append(.null) - } - } - - rows.append(row) - stepResult = sqlite3_step(statement) - } - - if !truncated, stepResult != SQLITE_DONE { - throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db))) - } - - if columns.isEmpty { - rowsAffected = Int(sqlite3_changes(db)) - } - - let executionTime = Date().timeIntervalSince(startTime) - - return SQLiteRawResult( - columns: columns, - columnTypeNames: columnTypeNames, - rows: rows, - rowsAffected: rowsAffected, - executionTime: executionTime, - isTruncated: truncated - ) - } -} - -private struct SQLiteRawResult: Sendable { - let columns: [String] - let columnTypeNames: [String] - let rows: [[PluginCellValue]] - let rowsAffected: Int - let executionTime: TimeInterval - let isTruncated: Bool -} - // MARK: - SQLite Plugin Driver final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let config: DriverConnectionConfig - private let busyState = SQLiteBusyState() - private let connectionActor: SQLiteConnectionActor - private let interruptLock = NSLock() - nonisolated(unsafe) private var _dbHandleForInterrupt: OpaquePointer? + private let backend: any SQLiteExecutionBackend private static let logger = Logger(subsystem: "com.TablePro", category: "SQLitePluginDriver") var currentSchema: String? { nil } - var serverVersion: String? { String(cString: sqlite3_libversion()) } + var serverVersion: String? { backend.resolvedServerVersion } var supportsSchemas: Bool { false } var supportsTransactions: Bool { true } @@ -576,30 +147,37 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { init(config: DriverConnectionConfig) { self.config = config - self.connectionActor = SQLiteConnectionActor(busyState: busyState) + self.backend = Self.makeBackend(config: config) + } + + /// A file-backed connection runs on the app's own SQLite; one whose transport marks it a remote + /// session runs on the server's SQLite through the agent. The mark and the token are set by the + /// app's transport when it rewrites the effective connection, never by the user. + private static func makeBackend(config: DriverConnectionConfig) -> any SQLiteExecutionBackend { + guard config.additionalFields[SQLiteAgentProtocol.backendFieldKey] == SQLiteAgentProtocol.agentBackendValue else { + return SQLiteLocalBackend(path: config.database) + } + return SQLiteAgentBackend( + host: config.host.isEmpty ? "127.0.0.1" : config.host, + port: config.port, + path: config.database, + token: config.additionalFields[SQLiteAgentProtocol.tokenFieldKey] ?? "" + ) } // MARK: - Connection func connect() async throws { - let path = expandPath(config.database) - - if !FileManager.default.fileExists(atPath: path) { - let directory = (path as NSString).deletingLastPathComponent - try? FileManager.default.createDirectory(atPath: directory, withIntermediateDirectories: true) + try await withTaskCancellationHandler { + try await backend.open() + } onCancel: { + backend.abortConnect() } - - try await connectionActor.open(path: path) - let rawHandle = await connectionActor.dbHandleForInterrupt - setInterruptHandle(rawHandle != 0 ? OpaquePointer(bitPattern: rawHandle) : nil) } func disconnect() { - interruptLock.lock() - _dbHandleForInterrupt = nil - interruptLock.unlock() - let actor = connectionActor - Task { await actor.close() } + let backend = self.backend + Task { await backend.close() } } func ping() async throws { @@ -607,13 +185,13 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func applyQueryTimeout(_ seconds: Int) async throws { - await connectionActor.applyBusyTimeout(Int32(max(0, seconds) * 1_000)) + await backend.applyBusyTimeout(Int32(max(0, seconds) * 1_000)) } // MARK: - Query Execution func execute(query: String) async throws -> PluginQueryResult { - let rawResult = try await connectionActor.executeQuery(query) + let rawResult = try await backend.executeQuery(query) return PluginQueryResult( columns: rawResult.columns, columnTypeNames: rawResult.columnTypeNames, @@ -625,7 +203,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - let rawResult = try await connectionActor.executeParameterizedQuery(query, parameters: parameters) + let rawResult = try await backend.executeParameterizedQuery(query, parameters: parameters) return PluginQueryResult( columns: rawResult.columns, columnTypeNames: rawResult.columnTypeNames, @@ -637,13 +215,10 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } /// `sqlite3_interrupt` ends a statement that is running. A connection waiting for a lock is - /// not running one, and measurably ignores it, so the busy handler is what ends that wait. + /// not running one, and measurably ignores it, so the busy handler is what ends that wait. The + /// remote backend forwards the same intent to the agent as a cancel frame. func cancelQuery() throws { - busyState.cancel() - interruptLock.lock() - defer { interruptLock.unlock() } - guard let db = _dbHandleForInterrupt else { return } - sqlite3_interrupt(db) + backend.canceller.cancel() } // MARK: - EXPLAIN @@ -1095,14 +670,6 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { """ } - // MARK: - Private Helpers - - nonisolated private func setInterruptHandle(_ handle: OpaquePointer?) { - interruptLock.lock() - _dbHandleForInterrupt = handle - interruptLock.unlock() - } - // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { @@ -1110,7 +677,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in let streamTask = Task { do { - try await self.connectionActor.streamQuery(queryToRun, continuation: continuation) + try await self.backend.streamQuery(queryToRun, continuation: continuation) } catch { continuation.finish(throwing: error) } @@ -1121,13 +688,6 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - private func expandPath(_ path: String) -> String { - if path.hasPrefix("~") { - return NSString(string: path).expandingTildeInPath - } - return path - } - // MARK: - Create Table DDL func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { diff --git a/TablePro/AppDelegate.swift b/TablePro/AppDelegate.swift index 88ec99047b..093d2d863c 100644 --- a/TablePro/AppDelegate.swift +++ b/TablePro/AppDelegate.swift @@ -196,6 +196,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { CloudflareTunnelManager.shared.terminateAllProcessesSync() CloudSQLProxyManager.shared.terminateAllProcessesSync() TunnelCommandManager.shared.terminateAllProcessesSync() + RemoteSQLiteTransportManager.shared.terminateAllProcessesSync() } private func persistOpenConnectionsForRecovery() { diff --git a/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift b/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift index 96338cb706..35cba785f3 100644 --- a/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift +++ b/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift @@ -65,8 +65,22 @@ struct ExecuteQueryChatTool: ChatTool { settings: mcpSettings ) - let tier = QueryClassifier.classifyTier(query, databaseType: meta.databaseType) - if tier == .destructive { + let classification = QueryClassifier.classify(query, databaseType: meta.databaseType) + + /// The same refusal MCP makes through `ExternalStatementGate`: a statement that reads or + /// writes files, or runs server-side code, is a strictly larger capability than reading and + /// writing data, and larger still on a connection whose transport executes on an SSH server. + /// The MCP tool routes through the gate; this path did not, so it added the reach that a + /// remote-execution connection turns into running code on the server. + if classification.reachesFilesystemOrExecutesCode { + return ChatToolResult( + content: "Statements that read or write files, or run server-side code (ATTACH, LOAD, COPY, VACUUM INTO), " + + "can't be sent from the assistant. Run this one in TablePro instead.", + isError: true + ) + } + + if classification.tier == .destructive { return ChatToolResult( content: "Destructive queries (DROP, TRUNCATE, ALTER...DROP) are blocked here. Use confirm_destructive_operation with the explicit confirmation phrase.", isError: true diff --git a/TablePro/Core/Database/DatabaseManager+RemoteSQLite.swift b/TablePro/Core/Database/DatabaseManager+RemoteSQLite.swift new file mode 100644 index 0000000000..835de5d925 --- /dev/null +++ b/TablePro/Core/Database/DatabaseManager+RemoteSQLite.swift @@ -0,0 +1,83 @@ +// +// DatabaseManager+RemoteSQLite.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +extension DatabaseManager { + /// Rewrites a SQLite connection to reach its database through a live session on the SSH server. + /// + /// The symmetry with the tunnel arm is the point: that one swaps `host` and `port` for a + /// forwarded port and hands the driver a connection with no idea a tunnel exists. This does the + /// same, and adds two fields the SQLite plugin reads to pick the agent backend and to present + /// the token the listener admits it with. The driver still just opens what it is told to open. + internal func buildRemoteSQLiteEffectiveConnection( + for connection: DatabaseConnection, + sshPasswordOverride: String? = nil + ) async throws -> DatabaseConnection { + let sshConfig = connection.resolvedSSHConfig + guard let field = pluginManager.localFilePathField(for: connection.type) else { + throw ConnectionTunnelError.remoteFileUnsupported(connection.type.displayName) + } + guard !sshConfig.remoteFilePath.isEmpty else { throw ConnectionTunnelError.remoteFilePathMissing } + + let credentials = sshCredentials(for: connection, passwordOverride: sshPasswordOverride) + let endpoint = try await RemoteSQLiteTransportManager.shared.createTunnel( + connectionId: connection.id, + config: sshConfig, + credentials: credentials + ) + + return connection.openingRemoteSQLiteSession( + host: "127.0.0.1", + port: endpoint.port, + token: endpoint.token, + remotePath: sshConfig.remoteFilePath, + field: field + ) + } + + /// Handle a dead remote SQLite session by reconnecting with exponential backoff, the same path + /// an SSH tunnel death takes. + func handleRemoteSQLiteTunnelDied(connectionId: UUID) async { + await recoverDeadTunnel( + connectionId: connectionId, + kind: "Remote SQLite", + disconnectedMessage: String(localized: "Remote SQLite session disconnected. Click to reconnect.") + ) + } +} + +extension DatabaseConnection { + /// Returns a copy whose driver opens the remote database over the loopback session: the file + /// path the agent opens, the local endpoint the driver dials, the marker that selects the agent + /// backend, and the token the listener admits it with. The SSH configuration is cleared so + /// nothing tries to tunnel a second time. + func openingRemoteSQLiteSession( + host: String, + port: Int, + token: String, + remotePath: String, + field: LocalFilePathField + ) -> DatabaseConnection { + var copy = self + copy.host = host + copy.port = port + switch field { + case .database: + copy.database = remotePath + case .additionalField(let id): + copy.additionalFields[id] = remotePath + } + copy.additionalFields[RemoteSQLiteWire.backendFieldKey] = RemoteSQLiteWire.agentBackendValue + copy.additionalFields[RemoteSQLiteWire.tokenFieldKey] = token + /// `resolvedSSHConfig` reads `sshTunnelMode`, so clearing `sshConfig` alone would leave the + /// effective connection still resolving to a remote session. Disabling the mode is what stops + /// anything from tunnelling the local endpoint a second time. + copy.sshConfig = SSHConfiguration() + copy.sshTunnelMode = .disabled + return copy + } +} diff --git a/TablePro/Core/Database/DatabaseManager+SSH.swift b/TablePro/Core/Database/DatabaseManager+SSH.swift index ee3e11d445..1f06f505c3 100644 --- a/TablePro/Core/Database/DatabaseManager+SSH.swift +++ b/TablePro/Core/Database/DatabaseManager+SSH.swift @@ -40,6 +40,11 @@ extension DatabaseManager { for: connection, sshPasswordOverride: sshPasswordOverride ) + case .remoteDatabaseSession: + return try await buildRemoteSQLiteEffectiveConnection( + for: connection, + sshPasswordOverride: sshPasswordOverride + ) case .ssh, .none: break } diff --git a/TablePro/Core/Database/DatabaseManager+SystemEvents.swift b/TablePro/Core/Database/DatabaseManager+SystemEvents.swift index c3375452d3..335ba531d2 100644 --- a/TablePro/Core/Database/DatabaseManager+SystemEvents.swift +++ b/TablePro/Core/Database/DatabaseManager+SystemEvents.swift @@ -60,6 +60,8 @@ extension DatabaseManager { await handleTunnelCommandDied(connectionId: connectionId) case .remoteFile: break + case .remoteDatabaseSession: + await handleRemoteSQLiteTunnelDied(connectionId: connectionId) } } } diff --git a/TablePro/Core/Database/DatabaseManager+Tunnel.swift b/TablePro/Core/Database/DatabaseManager+Tunnel.swift index 9a2b54e914..9ccadf03e6 100644 --- a/TablePro/Core/Database/DatabaseManager+Tunnel.swift +++ b/TablePro/Core/Database/DatabaseManager+Tunnel.swift @@ -101,6 +101,7 @@ extension DatabaseManager { case .socksProxy: return SOCKSProxyManager.shared case .tunnelCommand: return TunnelCommandManager.shared case .remoteFile: return RemoteFileTransportManager.shared + case .remoteDatabaseSession: return RemoteSQLiteTransportManager.shared case .none: return nil } } diff --git a/TablePro/Core/Database/DatabaseManager+Verification.swift b/TablePro/Core/Database/DatabaseManager+Verification.swift index e33c3ef870..eaec083e89 100644 --- a/TablePro/Core/Database/DatabaseManager+Verification.swift +++ b/TablePro/Core/Database/DatabaseManager+Verification.swift @@ -18,6 +18,10 @@ extension DatabaseManager { /// would only run a query against a database that cannot have gone away. internal func supportsHealthChecks(_ connectionId: UUID) -> Bool { guard let session = activeSessions[connectionId] else { return false } + /// A remote SQLite session holds a live socket to the server that can drop, so it is checked + /// even though the local SQLite engine, a file that cannot go away, is not. The answer is + /// per connection here rather than per type, because the same SQLite type is both. + if session.connection.opensRemoteDatabaseSession { return true } return PluginMetadataRegistry.shared.snapshot( for: session.connection.type )?.supportsHealthMonitor ?? true diff --git a/TablePro/Core/Plugins/PluginManager+Registration.swift b/TablePro/Core/Plugins/PluginManager+Registration.swift index 0cf9105532..abf4824304 100644 --- a/TablePro/Core/Plugins/PluginManager+Registration.swift +++ b/TablePro/Core/Plugins/PluginManager+Registration.swift @@ -541,6 +541,12 @@ extension PluginManager { .capabilities.supportsRemoteDatabaseFile ?? false } + /// Whether this type can run its statements on an SSH server against a live database file. + func supportsRemoteDatabaseSession(for databaseType: DatabaseType) -> Bool { + PluginMetadataRegistry.shared.snapshot(for: databaseType)? + .capabilities.supportsRemoteDatabaseSession ?? false + } + func supportsSSL(for databaseType: DatabaseType) -> Bool { PluginMetadataRegistry.shared.snapshot(for: databaseType)? .capabilities.supportsSSL ?? true diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift index 1490fe9d36..87abbefa66 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift @@ -668,7 +668,8 @@ extension PluginMetadataRegistry { supportsDatabaseTriggerBrowse: true, supportsCloudflareTunnel: false, localFilePathField: .database, - supportsRemoteDatabaseFile: true + supportsRemoteDatabaseFile: true, + supportsRemoteDatabaseSession: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index 089ce01315..4bbbdf6acd 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -101,6 +101,11 @@ struct PluginMetadataSnapshot: Sendable { /// out of it either fails to load or presents incomplete accounts, which is worse. var supportsRemoteDatabaseFile: Bool = false + /// Whether this type can run its statements on an SSH server against a live database file, + /// rather than only fetching a read-only copy. True for SQLite alone today, because the + /// remote agent opens the file with the server's `libsqlite3`. + var supportsRemoteDatabaseSession: Bool = false + var supportsPrincipalConnectionLimit: Bool = true static let defaults = CapabilityFlags( @@ -655,6 +660,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { localFilePathField: existingSnapshot?.capabilities.localFilePathField, supportsRemoteDatabaseFile: existingSnapshot?.capabilities .supportsRemoteDatabaseFile ?? false, + supportsRemoteDatabaseSession: existingSnapshot?.capabilities + .supportsRemoteDatabaseSession ?? false, supportsPrincipalConnectionLimit: existingSnapshot?.capabilities .supportsPrincipalConnectionLimit ?? true ), diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteAdmission.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteAdmission.swift new file mode 100644 index 0000000000..f371a3e9cd --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteAdmission.swift @@ -0,0 +1,48 @@ +// +// RemoteSQLiteAdmission.swift +// TablePro +// + +import Foundation + +/// Validates the line a client sends before any protocol frame, so only the driver the transport +/// handed the token to can reach the server's database through the loopback listener. +/// +/// Unlike a forwarded database port, whose database still authenticates the client, this listener +/// stands in front of a session that runs SQL with no further credential. The token is per +/// connection and random, and the comparison is constant time so a local process cannot recover it +/// one byte at a time. +enum RemoteSQLiteAdmission { + static let maxLineLength = 256 + + static func isAuthorized(line: Data, token: String) -> Bool { + guard !token.isEmpty, line.count <= maxLineLength else { return false } + guard let text = String(data: line, encoding: .utf8), text.hasPrefix(RemoteSQLiteWire.admissionPrefix) else { + return false + } + let presented = String(text.dropFirst(RemoteSQLiteWire.admissionPrefix.count)) + return constantTimeEquals(presented, token) + } + + static func constantTimeEquals(_ lhs: String, _ rhs: String) -> Bool { + let left = Array(lhs.utf8) + let right = Array(rhs.utf8) + guard left.count == right.count else { return false } + var difference: UInt8 = 0 + for index in 0.. String { + var bytes = [UInt8](repeating: 0, count: 32) + if SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) != errSecSuccess { + /// The system RNG effectively never fails on macOS, but an all-zero token on the one + /// path that would produce it is a guessable admission secret, so fall back to two + /// UUIDs rather than ship zeros. + return (UUID().uuidString + UUID().uuidString).replacingOccurrences(of: "-", with: "").lowercased() + } + return bytes.map { String(format: "%02x", $0) }.joined() + } +} diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift new file mode 100644 index 0000000000..88e2b025da --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift @@ -0,0 +1,481 @@ +// +// RemoteSQLiteAgentSource.swift +// TablePro +// + +import Foundation + +/// The Python program TablePro runs on the SSH server to open a SQLite database with the server's +/// own `libsqlite3`, and the shell command that launches it. +/// +/// The program is delivered inline over the SSH exec channel: base64 on the launcher's argument +/// list rather than a file written to the server, so nothing is installed and nothing is left +/// behind. It uses only the standard library, loads `libsqlite3` through `ctypes`, and speaks the +/// framed protocol in `SQLiteAgentProtocol`. It runs on the Python the server already carries; +/// measured across SQLite 3.26 to 3.54 and Python 3.6 to 3.14. +enum RemoteSQLiteAgentSource { + /// The interpreters the launcher tries, in order. The bare names resolve through the login + /// shell's `PATH`; the absolute paths cover RHEL/Rocky 8, whose only interpreter is + /// `platform-python` under `/usr/libexec` with no `python3` on `PATH`. + private static let interpreters = ["python3", "/usr/bin/python3", "/usr/libexec/platform-python"] + + /// The command handed to `libssh2_channel_process_startup("exec", …)`. + /// + /// The whole string is a fixed literal but for the base64 of the program, which is drawn from + /// the base64 alphabet alone and is therefore safe as a bare shell word and inside single + /// quotes under sh, bash, zsh, fish and csh (measured). The server never sees a byte of user + /// data on this line: the database path and every value travel as framed protocol fields once + /// the program is running, never as shell text. + /// + /// `-I` runs the interpreter isolated, so `PYTHON*` variables and the user site directory + /// cannot redirect it. When no interpreter is found the launcher prints the agent's own notice + /// on standard output, where it reaches the client as a launcher notice rather than being lost + /// on standard error, and exits cleanly. + static func launcherCommand() -> String { + let encoded = Data(python.utf8).base64EncodedString() + let loop = interpreters.map { interpreter -> String in + let present = #"command -v "\#(interpreter)" >/dev/null 2>&1"# + let run = #"exec "\#(interpreter)" -I -S -c "import base64,sys;exec(base64.b64decode(sys.argv[1]))" \#(encoded)"# + return "if \(present); then \(run); fi" + } + .joined(separator: "; ") + return "sh -c '" + loop + "; echo " + RemoteSQLiteWire.noPythonNotice + "; exit 0'" + } + + static let python = #""" +import sys + +if sys.version_info[0] < 3: + sys.exit(2) + +import ctypes +import ctypes.util +import os +import struct +import threading +import time + +PROTOCOL_VERSION = 1 + +OP_HELLO = 0x01 +OP_EXECUTE = 0x02 +OP_CANCEL = 0x03 +OP_HEARTBEAT = 0x04 +OP_SET_BUSY = 0x05 +OP_READY = 0x81 +OP_FAILURE = 0x82 +OP_HEADER = 0x83 +OP_ROWS = 0x84 +OP_DONE = 0x85 +OP_ERROR = 0x86 + +FAIL_UNSUPPORTED_PROTOCOL = 1 +FAIL_PYTHON_TOO_OLD = 2 +FAIL_LIBRARY_UNAVAILABLE = 3 +FAIL_OPEN_FAILED = 4 +FAIL_CALLBACKS_UNAVAILABLE = 5 +FAIL_MALFORMED = 6 + +SQLITE_OK = 0 +SQLITE_DENY = 1 +SQLITE_ROW = 100 +SQLITE_DONE = 101 +SQLITE_FUNCTION = 31 +SQLITE_OPEN_READWRITE = 0x00000002 +SQLITE_COL_NULL = 5 +SQLITE_COL_BLOB = 4 + +DENIED_FUNCTIONS = (b"fts3_tokenizer", b"load_extension") +IDLE_DEADLINE_SECONDS = 60.0 +BUSY_RETRY_MS = 10 +ROW_BATCH = 500 +BATCH_BYTES = 256 * 1024 +MAX_FRAME = 64 * 1024 * 1024 + +stdout = sys.stdout.buffer +stdin = sys.stdin.buffer +write_lock = threading.Lock() +last_activity = [time.time()] +cancel_flag = [False] +busy_timeout_ms = [0] + + +def load_sqlite(): + names = ["libsqlite3.so.0", "libsqlite3.so", "libsqlite3.dylib"] + found = ctypes.util.find_library("sqlite3") + if found: + names.append(found) + for name in names: + try: + return ctypes.CDLL(name) + except OSError: + continue + return None + + +def configure(lib): + p = ctypes.c_void_p + lib.sqlite3_libversion.restype = ctypes.c_char_p + lib.sqlite3_open_v2.argtypes = [ctypes.c_char_p, ctypes.POINTER(p), ctypes.c_int, ctypes.c_char_p] + lib.sqlite3_errmsg.argtypes = [p] + lib.sqlite3_errmsg.restype = ctypes.c_char_p + lib.sqlite3_extended_result_codes.argtypes = [p, ctypes.c_int] + lib.sqlite3_set_authorizer.argtypes = [p, p, p] + lib.sqlite3_busy_handler.argtypes = [p, p, p] + lib.sqlite3_prepare_v2.argtypes = [p, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(p), ctypes.POINTER(ctypes.c_char_p)] + lib.sqlite3_column_count.argtypes = [p] + lib.sqlite3_column_name.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_name.restype = ctypes.c_char_p + lib.sqlite3_column_decltype.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_decltype.restype = ctypes.c_char_p + lib.sqlite3_bind_null.argtypes = [p, ctypes.c_int] + lib.sqlite3_bind_text.argtypes = [p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, p] + lib.sqlite3_bind_blob.argtypes = [p, ctypes.c_int, p, ctypes.c_int, p] + lib.sqlite3_step.argtypes = [p] + lib.sqlite3_column_type.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_bytes.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_blob.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_blob.restype = p + lib.sqlite3_column_text.argtypes = [p, ctypes.c_int] + lib.sqlite3_column_text.restype = p + lib.sqlite3_changes.argtypes = [p] + lib.sqlite3_finalize.argtypes = [p] + lib.sqlite3_reset.argtypes = [p] + lib.sqlite3_interrupt.argtypes = [p] + lib.sqlite3_close_v2.argtypes = [p] + + +def frame(op, payload): + return struct.pack(">I", len(payload) + 1) + bytes([op]) + payload + + +def send(op, payload): + with write_lock: + stdout.write(frame(op, payload)) + stdout.flush() + + +def send_failure(code, message): + send(OP_FAILURE, struct.pack(">I", code) + encode_str(message)) + + +def encode_bytes(data): + return struct.pack(">I", len(data)) + data + + +def encode_str(text): + return encode_bytes(text.encode("utf-8")) + + +def encode_value(kind, data): + if kind == 0: + return bytes([0]) + return bytes([kind]) + encode_bytes(data) + + +def read_exact(count): + chunks = [] + remaining = count + while remaining > 0: + chunk = stdin.read(remaining) + if not chunk: + return None + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def read_frame(): + header = read_exact(4) + if header is None: + return None + length = struct.unpack(">I", header)[0] + if length < 1 or length > MAX_FRAME: + return None + body = read_exact(length) + if body is None: + return None + return body[0], body[1:] + + +class Cursor(object): + def __init__(self, data): + self.data = data + self.offset = 0 + + def u8(self): + value = self.data[self.offset] + self.offset += 1 + return value + + def u32(self): + value = struct.unpack(">I", self.data[self.offset:self.offset + 4])[0] + self.offset += 4 + return value + + def blob(self): + length = self.u32() + value = self.data[self.offset:self.offset + length] + self.offset += length + return value + + def text(self): + return self.blob().decode("utf-8") + + def value(self): + tag = self.u8() + if tag == 0: + return (0, None) + return (tag, self.blob()) + + +def make_authorizer(): + authz_type = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.c_int, + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p + ) + + def authorizer(_ctx, action, arg1, arg2, _db, _trigger): + if action == SQLITE_FUNCTION and arg2 is not None and arg2.lower() in DENIED_FUNCTIONS: + return SQLITE_DENY + return SQLITE_OK + + return authz_type(authorizer) + + +def make_busy_handler(lib, db): + busy_type = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_int) + + def handler(_ctx, count): + if cancel_flag[0]: + return 0 + limit = busy_timeout_ms[0] + if limit > 0 and count * BUSY_RETRY_MS >= limit: + return 0 + time.sleep(BUSY_RETRY_MS / 1000.0) + return 1 + + return busy_type(handler) + + +def open_database(lib, path, busy_ms): + db = ctypes.c_void_p() + expanded = os.path.expanduser(path) + rc = lib.sqlite3_open_v2(expanded.encode("utf-8"), ctypes.byref(db), SQLITE_OPEN_READWRITE, None) + if rc != SQLITE_OK: + message = "unable to open database" + if db.value: + message = lib.sqlite3_errmsg(db).decode("utf-8", "replace") + lib.sqlite3_close_v2(db) + return None, message + lib.sqlite3_extended_result_codes(db, 1) + busy_timeout_ms[0] = busy_ms + return db, None + + +TRANSIENT = ctypes.cast(-1, ctypes.c_void_p) + + +def bind_parameters(lib, stmt, parameters): + for index, item in enumerate(parameters): + position = index + 1 + tag, data = item + if tag == 0: + lib.sqlite3_bind_null(stmt, position) + elif tag == 1: + lib.sqlite3_bind_text(stmt, position, data, len(data), TRANSIENT) + else: + buffer = ctypes.create_string_buffer(data, len(data)) if data else ctypes.create_string_buffer(0) + lib.sqlite3_bind_blob(stmt, position, buffer, len(data), TRANSIENT) + + +def read_row(lib, stmt, column_count): + row = [] + for index in range(column_count): + column_type = lib.sqlite3_column_type(stmt, index) + if column_type == SQLITE_COL_NULL: + row.append((0, b"")) + elif column_type == SQLITE_COL_BLOB: + length = lib.sqlite3_column_bytes(stmt, index) + pointer = lib.sqlite3_column_blob(stmt, index) + data = ctypes.string_at(pointer, length) if length and pointer else b"" + row.append((2, data)) + else: + pointer = lib.sqlite3_column_text(stmt, index) + length = lib.sqlite3_column_bytes(stmt, index) + data = ctypes.string_at(pointer, length) if length and pointer else b"" + row.append((1, data)) + return row + + +def execute(lib, db, sql, parameters, row_cap): + cancel_flag[0] = False + stmt = ctypes.c_void_p() + rc = lib.sqlite3_prepare_v2(db, sql.encode("utf-8"), -1, ctypes.byref(stmt), None) + if rc != SQLITE_OK or not stmt.value: + if stmt.value: + lib.sqlite3_finalize(stmt) + send(OP_ERROR, struct.pack(">I", rc & 0xFFFFFFFF) + encode_str(lib.sqlite3_errmsg(db).decode("utf-8", "replace"))) + return + try: + bind_parameters(lib, stmt, parameters) + column_count = lib.sqlite3_column_count(stmt) + header = struct.pack(">I", column_count) + for index in range(column_count): + name = lib.sqlite3_column_name(stmt, index) + header += encode_str(name.decode("utf-8", "replace") if name else "column_" + str(index)) + decltype = lib.sqlite3_column_decltype(stmt, index) + if decltype is not None: + header += bytes([1]) + encode_str(decltype.decode("utf-8", "replace")) + else: + header += bytes([0]) + send(OP_HEADER, header) + + batch = [] + batch_bytes = 0 + row_count = 0 + truncated = False + while True: + step = lib.sqlite3_step(stmt) + if step != SQLITE_ROW: + break + if row_cap > 0 and row_count >= row_cap: + truncated = True + break + row = read_row(lib, stmt, column_count) + batch.append(row) + row_count += 1 + for _, data in row: + batch_bytes += len(data) + if len(batch) >= ROW_BATCH or batch_bytes >= BATCH_BYTES: + send_rows(column_count, batch) + batch = [] + batch_bytes = 0 + if batch: + send_rows(column_count, batch) + if step != SQLITE_ROW and step != SQLITE_DONE and not truncated: + send(OP_ERROR, struct.pack(">I", step & 0xFFFFFFFF) + encode_str(lib.sqlite3_errmsg(db).decode("utf-8", "replace"))) + return + changes = lib.sqlite3_changes(db) if column_count == 0 else 0 + send(OP_DONE, struct.pack(">q", changes) + bytes([1 if truncated else 0])) + finally: + lib.sqlite3_finalize(stmt) + + +def send_rows(column_count, batch): + payload = struct.pack(">I", column_count) + struct.pack(">I", len(batch)) + parts = [payload] + for row in batch: + for kind, data in row: + parts.append(encode_value(kind, data)) + send(OP_ROWS, b"".join(parts)) + + +def reader_loop(lib, db, request_queue, condition): + while True: + message = read_frame() + last_activity[0] = time.time() + if message is None: + with condition: + request_queue.append(None) + condition.notify() + return + op, payload = message + if op == OP_CANCEL: + cancel_flag[0] = True + lib.sqlite3_interrupt(db) + elif op == OP_HEARTBEAT: + continue + elif op == OP_SET_BUSY: + busy_timeout_ms[0] = Cursor(payload).u32() + else: + with condition: + request_queue.append((op, payload)) + condition.notify() + + +def watchdog(lib, db): + while True: + time.sleep(5.0) + if time.time() - last_activity[0] > IDLE_DEADLINE_SECONDS: + lib.sqlite3_interrupt(db) + os._exit(0) + + +def main(): + if sys.version_info < (3, 6): + send_failure(FAIL_PYTHON_TOO_OLD, "python 3.6 or newer is required") + return + + lib = load_sqlite() + if lib is None: + send_failure(FAIL_LIBRARY_UNAVAILABLE, "libsqlite3 could not be loaded on the server") + return + configure(lib) + + hello = read_frame() + last_activity[0] = time.time() + if hello is None or hello[0] != OP_HELLO: + send_failure(FAIL_MALFORMED, "expected HELLO") + return + cursor = Cursor(hello[1]) + client_version = cursor.u32() + path = cursor.text() + busy_ms = cursor.u32() + if client_version != PROTOCOL_VERSION: + send_failure(FAIL_UNSUPPORTED_PROTOCOL, "protocol version mismatch") + return + + db, error = open_database(lib, path, busy_ms) + if db is None: + send_failure(FAIL_OPEN_FAILED, error) + return + + authorizer = make_authorizer() + busy_handler = make_busy_handler(lib, db) + keep_alive = (authorizer, busy_handler) + try: + lib.sqlite3_set_authorizer(db, ctypes.cast(authorizer, ctypes.c_void_p), None) + lib.sqlite3_busy_handler(db, ctypes.cast(busy_handler, ctypes.c_void_p), None) + probe = ctypes.c_void_p() + rc = lib.sqlite3_prepare_v2(db, b"SELECT 1", -1, ctypes.byref(probe), None) + if rc != SQLITE_OK: + raise RuntimeError("callback self-test failed") + lib.sqlite3_finalize(probe) + except Exception: + send_failure(FAIL_CALLBACKS_UNAVAILABLE, "the server rejected the required callbacks") + return + + send(OP_READY, struct.pack(">I", PROTOCOL_VERSION) + + encode_str(lib.sqlite3_libversion().decode("utf-8", "replace")) + + encode_str("%d.%d.%d" % sys.version_info[:3])) + + request_queue = [] + condition = threading.Condition() + threading.Thread(target=reader_loop, args=(lib, db, request_queue, condition), daemon=True).start() + threading.Thread(target=watchdog, args=(lib, db), daemon=True).start() + + _ = keep_alive + while True: + with condition: + while not request_queue: + condition.wait() + item = request_queue.pop(0) + if item is None: + break + op, payload = item + if op != OP_EXECUTE: + continue + request = Cursor(payload) + sql = request.text() + count = request.u32() + parameters = [request.value() for _ in range(count)] + row_cap = request.u32() + execute(lib, db, sql, parameters, row_cap) + + lib.sqlite3_close_v2(db) + + +main() +"""# +} diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift new file mode 100644 index 0000000000..95454e07ac --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift @@ -0,0 +1,113 @@ +// +// RemoteSQLiteExecChannelOpener.swift +// TablePro +// + +import CLibSSH2 +import Foundation + +/// Opens one session channel and starts a command on it, one non-blocking attempt at a time, so +/// `SSHForwardChannelOpenPump` can drive it to a decision within an app-owned deadline exactly as it +/// drives a forwarding open. A forwarding channel opens in a single call; an exec channel is two +/// steps, the channel open and the `exec` request, and either can report `LIBSSH2_ERROR_EAGAIN`, so +/// the phase is remembered between attempts. +/// +/// The opener owns the channel until it hands it back as `.opened`. If the pump times out or is +/// cancelled after the channel was created but before it was handed over, the caller must call +/// `abort()` to free it, because dropping the Swift reference does not free the libssh2 channel and +/// the session stays live. +final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { + private enum Phase { + case opening + case starting + case configuring + case finished + } + + private let session: OpaquePointer + private let command: String + private let sessionQueue: DispatchQueue + private var phase: Phase = .opening + private var channel: OpaquePointer? + + init(session: OpaquePointer, command: String, sessionQueue: DispatchQueue) { + self.session = session + self.command = command + self.sessionQueue = sessionQueue + } + + func attemptOpen() -> SSHForwardChannelAttempt { + sessionQueue.sync { + switch phase { + case .opening: + return openChannel() + case .starting: + return startCommand() + case .configuring: + return configure() + case .finished: + return .failed(code: 0, message: "channel open already finished") + } + } + } + + /// Frees a channel that was opened but never handed to the relay, for an open the pump timed out + /// or cancelled. A no-op once the channel has been handed over or already freed. + func abort() { + sessionQueue.sync { freeChannel() } + } + + private func openChannel() -> SSHForwardChannelAttempt { + if let opened = libssh2_channel_open_ex(session, "session", 7, 2 * 1_024 * 1_024, 32_768, nil, 0) { + channel = opened + phase = .starting + return startCommand() + } + return wouldBlockOrFailed() + } + + private func startCommand() -> SSHForwardChannelAttempt { + guard let channel else { return .failed(code: 0, message: "no channel to start a command on") } + let rc = libssh2_channel_process_startup(channel, "exec", 4, command, UInt32(command.utf8.count)) + if rc == 0 { + phase = .configuring + return configure() + } + if rc == LIBSSH2_ERROR_EAGAIN { + return .wouldBlock(RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session))) + } + let message = LibSSH2ForwardChannel.lastErrorMessage(session: session) + freeChannel() + return .failed(code: rc, message: message) + } + + /// Discards the agent's standard error rather than merging it into the protocol stream. A merge + /// would corrupt the framed replies; leaving it unread would let a large Python traceback fill + /// the channel window and stall the relay, which reads standard output alone. + private func configure() -> SSHForwardChannelAttempt { + guard let channel else { return .failed(code: 0, message: "no channel to configure") } + let rc = libssh2_channel_handle_extended_data2(channel, LIBSSH2_CHANNEL_EXTENDED_DATA_IGNORE) + if rc == LIBSSH2_ERROR_EAGAIN { + return .wouldBlock(RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session))) + } + self.channel = nil + phase = .finished + return .opened(channel) + } + + private func wouldBlockOrFailed() -> SSHForwardChannelAttempt { + let errorCode = libssh2_session_last_errno(session) + guard errorCode == LIBSSH2_ERROR_EAGAIN else { + return .failed(code: errorCode, message: LibSSH2ForwardChannel.lastErrorMessage(session: session)) + } + return .wouldBlock(RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session))) + } + + private func freeChannel() { + if let channel { + libssh2_channel_free(channel) + self.channel = nil + } + phase = .finished + } +} diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift new file mode 100644 index 0000000000..c3f73e6443 --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift @@ -0,0 +1,158 @@ +// +// RemoteSQLiteTransportManager.swift +// TablePro +// + +import Foundation +import os + +/// The loopback endpoint a remote SQLite connection's driver dials, and the token that admits it. +struct RemoteSQLiteEndpoint: Sendable { + let port: Int + let token: String +} + +/// Owns one `RemoteSQLiteTunnel` per connection, so disconnect, health checks and the +/// mutual-exclusivity rule reach a live remote SQLite session the way they reach every other +/// transport. +actor RemoteSQLiteTransportManager: TunnelManaging { + static let shared = RemoteSQLiteTransportManager() + + private static let logger = Logger(subsystem: "com.TablePro", category: "RemoteSQLiteTransport") + private static let portRangeStart = 60_000 + private static let portRangeEnd = 65_000 + + private var tunnels: [UUID: RemoteSQLiteTunnel] = [:] + private static let tunnelRegistry = OSAllocatedUnfairLock(initialState: [UUID: RemoteSQLiteTunnel]()) + private var appNapActivity: NSObjectProtocol? + + private init() {} + + func createTunnel( + connectionId: UUID, + config: SSHConfiguration, + credentials: SSHTunnelCredentials + ) async throws -> RemoteSQLiteEndpoint { + if tunnels[connectionId] != nil { + try await closeTunnel(connectionId: connectionId) + } + + let token = RemoteSQLiteAdmission.newToken() + let command = RemoteSQLiteAgentSource.launcherCommand() + let candidatePorts = localPortCandidates() + + /// The tunnel is built inside the detached task and only it, not the authenticated chain of + /// raw libssh2 handles it holds, crosses back to the actor. That is what `LibSSH2Tunnel`'s + /// factory does too, and it is what keeps the non-Sendable session out of an actor hop. + let tunnel = try await Task.detached { + let chain = try await LibSSH2TunnelFactory.buildAuthenticatedChain( + config: config, + credentials: credentials, + queueLabel: "com.TablePro.rsqlite.hop.\(connectionId.uuidString)" + ) + for port in candidatePorts { + if let listenFD = Self.bindLoopbackListener(port: port) { + return RemoteSQLiteTunnel( + connectionId: connectionId, + localPort: port, + chain: chain, + listenFD: listenFD, + command: command, + token: token + ) + } + } + LibSSH2TunnelFactory.cleanupChain(chain, reason: "no local port") + throw SSHTunnelError.noAvailablePort + }.value + + tunnel.onDeath = { [weak self] id in + Task { [weak self] in await self?.handleTunnelDeath(connectionId: id) } + } + + tunnels[connectionId] = tunnel + Self.tunnelRegistry.withLock { $0[connectionId] = tunnel } + tunnel.startForwarding() + tunnel.startKeepAlive() + updateAppNapState() + + Self.logger.info("Remote SQLite session ready for \(connectionId) on 127.0.0.1:\(tunnel.localPort)") + return RemoteSQLiteEndpoint(port: tunnel.localPort, token: token) + } + + func closeTunnel(connectionId: UUID) async throws { + guard let tunnel = tunnels.removeValue(forKey: connectionId) else { return } + Self.tunnelRegistry.withLock { $0[connectionId] = nil } + updateAppNapState() + tunnel.close() + } + + func hasTunnel(connectionId: UUID) -> Bool { + guard let tunnel = tunnels[connectionId] else { return false } + return tunnel.isRunning + } + + func getLocalPort(connectionId: UUID) -> Int? { + guard let tunnel = tunnels[connectionId], tunnel.isRunning else { return nil } + return tunnel.localPort + } + + nonisolated func terminateAllProcessesSync() { + let tunnelsToClose = Self.tunnelRegistry.withLock { dict -> [RemoteSQLiteTunnel] in + let values = Array(dict.values) + dict.removeAll() + return values + } + for tunnel in tunnelsToClose { tunnel.closeSync() } + } + + // MARK: - Private + + private func handleTunnelDeath(connectionId: UUID) async { + guard tunnels.removeValue(forKey: connectionId) != nil else { return } + Self.tunnelRegistry.withLock { $0[connectionId] = nil } + updateAppNapState() + Self.logger.warning("Remote SQLite session died for \(connectionId)") + await DatabaseManager.shared.handleRemoteSQLiteTunnelDied(connectionId: connectionId) + } + + private func localPortCandidates() -> [Int] { + Array(Self.portRangeStart...Self.portRangeEnd).shuffled() + } + + private static func bindLoopbackListener(port: Int) -> Int32? { + let listenFD = socket(AF_INET, SOCK_STREAM, 0) + guard listenFD >= 0 else { return nil } + + var reuseAddr: Int32 = 1 + setsockopt(listenFD, SOL_SOCKET, SO_REUSEADDR, &reuseAddr, socklen_t(MemoryLayout.size)) + + var addr = sockaddr_in() + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = in_port_t(port).bigEndian + addr.sin_addr.s_addr = inet_addr("127.0.0.1") + + let bound = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(listenFD, $0, socklen_t(MemoryLayout.size)) + } + } + guard bound == 0, listen(listenFD, 16) == 0 else { + Darwin.close(listenFD) + return nil + } + return listenFD + } + + private func updateAppNapState() { + if !tunnels.isEmpty && appNapActivity == nil { + appNapActivity = ProcessInfo.processInfo.beginActivity( + options: .userInitiatedAllowingIdleSystemSleep, + reason: "Remote SQLite session keepalive requires timely execution" + ) + } else if tunnels.isEmpty, let activity = appNapActivity { + ProcessInfo.processInfo.endActivity(activity) + appNapActivity = nil + } + } +} diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift new file mode 100644 index 0000000000..0487ded048 --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift @@ -0,0 +1,293 @@ +// +// RemoteSQLiteTunnel.swift +// TablePro +// + +import CLibSSH2 +import Foundation +import os + +/// A loopback listener whose accepted clients each get a fresh exec channel running the SQLite +/// agent on the SSH server, so the SQLite plugin can reach a live database on that server as if it +/// were a local port. +/// +/// It mirrors `LibSSH2Tunnel`, which forwards a TCP port over one authenticated session: the same +/// accept loop, keepalive, teardown latch and byte counter. What differs is the cargo. A forwarding +/// tunnel opens a direct-tcpip channel per client; this opens a session channel and runs a command +/// on it, and it admits a client only after the client presents the per-connection token, because +/// nothing behind this listener authenticates the way a database port does. +final class RemoteSQLiteTunnel: @unchecked Sendable { + let connectionId: UUID + let localPort: Int + let createdAt: Date + + private static let logger = Logger(subsystem: "com.TablePro", category: "RemoteSQLiteTunnel") + + private let chain: LibSSH2TunnelFactory.AuthenticatedChain + private let listenFD: Int32 + private let command: String + private let token: String + + private var session: OpaquePointer { chain.session } + private var socketFD: Int32 { chain.socketFD } + + private var forwardingTask: Task? + private var keepAliveTask: Task? + private let aliveLatch = TeardownLatch() + private let clientTasks = OSAllocatedUnfairLock(initialState: [Task]()) + + private let sessionQueue: DispatchQueue + private let relayQueue: DispatchQueue + private let acceptQueue: DispatchQueue + + var onDeath: ((UUID) -> Void)? + + private let byteCounter = TransportByteCounter() + + private static let relayBufferSize = 32_768 + private static let channelOpenDeadlineSeconds: TimeInterval = 6 + private static let channelOpenPollTimeoutMs: Int32 = 5_000 + private static let acceptPollTimeoutMs: Int32 = 200 + private static let admissionTimeoutSeconds = 5 + + init( + connectionId: UUID, + localPort: Int, + chain: LibSSH2TunnelFactory.AuthenticatedChain, + listenFD: Int32, + command: String, + token: String + ) { + self.connectionId = connectionId + self.localPort = localPort + self.chain = chain + self.listenFD = listenFD + self.command = command + self.token = token + self.createdAt = Date() + self.sessionQueue = DispatchQueue(label: "com.TablePro.rsqlite.session.\(connectionId.uuidString)", qos: .utility) + self.relayQueue = DispatchQueue( + label: "com.TablePro.rsqlite.relay.\(connectionId.uuidString)", qos: .utility, attributes: .concurrent + ) + self.acceptQueue = DispatchQueue(label: "com.TablePro.rsqlite.accept.\(connectionId.uuidString)", qos: .utility) + TransportActivityRegistry.shared.register(byteCounter, for: connectionId) + } + + var isRunning: Bool { aliveLatch.isLive } + + // MARK: - Forwarding + + func startForwarding() { + sessionQueue.sync { libssh2_session_set_blocking(session, 0) } + + forwardingTask = Task.detached { [weak self] in + guard let self else { return } + await withCheckedContinuation { (continuation: CheckedContinuation) in + self.acceptQueue.async { [weak self] in + defer { continuation.resume() } + guard let self else { return } + Self.logger.info("Remote SQLite listener started on 127.0.0.1:\(self.localPort)") + while self.isRunning { + guard let clientFD = self.acceptClient() else { + if self.isRunning { continue } + break + } + self.spawnClient(clientFD: clientFD) + } + } + } + } + } + + func startKeepAlive() { + sessionQueue.sync { libssh2_keepalive_config(session, 1, 30) } + keepAliveTask = Task.detached { [weak self] in + guard let self else { return } + while !Task.isCancelled && self.isRunning { + let failed = await withCheckedContinuation { (continuation: CheckedContinuation) in + self.sessionQueue.async { + var secondsToNext: Int32 = 0 + let rc = libssh2_keepalive_send(self.session, &secondsToNext) + continuation.resume(returning: sshKeepAliveDidFail(rc)) + } + } + if failed { + Self.logger.warning("Remote SQLite keepalive failed, marking dead") + self.markDead() + break + } + try? await Task.sleep(for: .seconds(10)) + } + } + } + + // MARK: - Lifecycle + + func close() { + guard aliveLatch.claim() else { return } + performTeardown() + } + + func closeSync() { + guard aliveLatch.claim() else { return } + forwardingTask?.cancel() + keepAliveTask?.cancel() + clientTasks.withLock { tasks in + for task in tasks { task.cancel() } + tasks.removeAll() + } + shutdown(socketFD, SHUT_RDWR) + Darwin.close(listenFD) + } + + private func markDead() { + guard aliveLatch.claim() else { return } + performTeardown() + onDeath?(connectionId) + } + + private func performTeardown() { + forwardingTask?.cancel() + keepAliveTask?.cancel() + let currentClientTasks = clientTasks.withLock { tasks -> [Task] in + let copy = tasks + for task in tasks { task.cancel() } + tasks.removeAll() + return copy + } + + shutdown(socketFD, SHUT_RDWR) + Darwin.close(listenFD) + + let chain = self.chain + let forwardingTask = self.forwardingTask + let keepAliveTask = self.keepAliveTask + let connectionId = self.connectionId + Task.detached { + await forwardingTask?.value + await keepAliveTask?.value + for task in currentClientTasks { await task.value } + LibSSH2TunnelFactory.cleanupChain(chain, reason: "Closing remote SQLite session") + Self.logger.info("Remote SQLite session closed for \(connectionId)") + } + } + + // MARK: - Accept + + private func acceptClient() -> Int32? { + var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, Self.acceptPollTimeoutMs) > 0, pollFD.revents & Int16(POLLIN) != 0 else { + return nil + } + var clientAddr = sockaddr_in() + var addrLen = socklen_t(MemoryLayout.size) + let clientFD = withUnsafeMutablePointer(to: &clientAddr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { accept(listenFD, $0, &addrLen) } + } + return clientFD >= 0 ? clientFD : nil + } + + private func spawnClient(clientFD: Int32) { + let task = Task.detached { [weak self] in + guard let self else { Darwin.close(clientFD); return } + await withCheckedContinuation { (continuation: CheckedContinuation) in + self.relayQueue.async { [weak self] in + defer { continuation.resume() } + guard let self else { Darwin.close(clientFD); return } + self.admitAndRelay(clientFD: clientFD) + } + } + } + let shouldCancel = clientTasks.withLock { tasks -> Bool in + tasks.removeAll { $0.isCancelled } + tasks.append(task) + return !aliveLatch.isLive + } + if shouldCancel { task.cancel() } + } + + private func admitAndRelay(clientFD: Int32) { + guard let line = Self.readAdmissionLine(clientFD), + RemoteSQLiteAdmission.isAuthorized(line: line, token: token) else { + Self.logger.warning("Remote SQLite client rejected: bad or missing token") + Darwin.close(clientFD) + return + } + + let opener = RemoteSQLiteExecChannelOpener(session: session, command: command, sessionQueue: sessionQueue) + let pump = SSHForwardChannelOpenPump( + opener: opener, + isActive: { [weak self] in self?.isRunning ?? false }, + deadline: Date().addingTimeInterval(Self.channelOpenDeadlineSeconds), + pollForReadiness: { [weak self] directions in + guard let self else { return false } + return pollReady(fd: self.socketFD, directions: directions, timeoutMs: Self.channelOpenPollTimeoutMs) + } + ) + + switch pump.run() { + case .opened(let channel): + runRelay(clientFD: clientFD, channel: channel) + case .failed(let code, let message): + Self.logger.error("Remote SQLite exec channel failed to open, libssh2 \(code): \(message)") + Darwin.close(clientFD) + case .timedOut, .cancelled: + /// The channel may have opened before the deadline and be waiting on the exec request. + /// The pump never took ownership, so free it here rather than leak it on the live session. + opener.abort() + Darwin.close(clientFD) + } + } + + /// Reads the token line the client sends before any protocol frame. One byte at a time up to the + /// newline, so the first protocol bytes that follow it stay in the socket for the relay, with a + /// receive timeout and a length cap so a silent or hostile client cannot hold the slot open. + private static func readAdmissionLine(_ clientFD: Int32) -> Data? { + var timeout = timeval(tv_sec: admissionTimeoutSeconds, tv_usec: 0) + setsockopt(clientFD, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + defer { + var clear = timeval(tv_sec: 0, tv_usec: 0) + setsockopt(clientFD, SOL_SOCKET, SO_RCVTIMEO, &clear, socklen_t(MemoryLayout.size)) + } + + var line = Data() + var byte: UInt8 = 0 + while line.count <= RemoteSQLiteAdmission.maxLineLength { + let read = recv(clientFD, &byte, 1, 0) + if read <= 0 { return nil } + if byte == 0x0A { return line } + line.append(byte) + } + return nil + } + + private func runRelay(clientFD: Int32, channel: OpaquePointer) { + let relay = SSHChannelRelay( + localFD: clientFD, + transportFD: socketFD, + channelIO: LibSSH2ChannelIO(channel: channel, session: session, sessionQueue: sessionQueue), + bufferSize: Self.relayBufferSize, + isActive: { [weak self] in self?.isRunning ?? false }, + byteCounter: byteCounter + ) + + let termination = relay.run() + Darwin.close(clientFD) + + guard isRunning else { return } + + /// Send EOF so the agent sees its standard input close and exits, then free without the + /// blocking `libssh2_channel_close`, which waits for the remote child to exit and would + /// stall every other call on this session's serial queue. The agent's own idle watchdog + /// reaps it if EOF is somehow missed. + sessionQueue.sync { + _ = libssh2_channel_send_eof(channel) + libssh2_channel_free(channel) + } + + if termination == .transportHangup { + Self.logger.info("Remote SQLite transport hung up for \(self.connectionId)") + markDead() + } + } +} diff --git a/TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift b/TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift new file mode 100644 index 0000000000..d3af6304ac --- /dev/null +++ b/TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift @@ -0,0 +1,34 @@ +// +// RemoteSQLiteWire.swift +// TablePro +// + +import Foundation + +/// The handful of wire constants the app needs to launch and admit a remote SQLite agent. +/// +/// The agent's own copy lives in the SQLite plugin (`SQLiteAgentProtocol`), which owns the full +/// codec. These two sets have to agree, and nothing at compile time forces a plugin in a separate +/// module to match the app; `RemoteSQLiteWireParityTests` reads both and fails on any drift. +enum RemoteSQLiteWire { + /// Written into the effective connection so the SQLite driver picks the agent backend rather + /// than opening a local file. + static let backendFieldKey = "sqliteBackend" + static let agentBackendValue = "agent" + + /// Carries the per-connection admission token to the driver. Never logged. + static let tokenFieldKey = "sqliteAgentToken" + + /// The first line a client sends to the loopback listener, before any protocol frame, so a + /// local process that has not been handed the token cannot reach the server's database. + static let admissionPrefix = "TPRSQL1 " + + /// Printed by the launcher on standard output, where it reaches the client as a launcher notice, + /// when the server carries no usable Python. Standard error is discarded, so the notice rides + /// the same path the protocol does. + static let noPythonNotice = "TPRSQL:NO_PYTHON" + + static func admissionLine(token: String) -> Data { + Data((admissionPrefix + token + "\n").utf8) + } +} diff --git a/TablePro/Models/Connection/ConnectionStageLabelFormatter.swift b/TablePro/Models/Connection/ConnectionStageLabelFormatter.swift index dd693d5f39..475aae264a 100644 --- a/TablePro/Models/Connection/ConnectionStageLabelFormatter.swift +++ b/TablePro/Models/Connection/ConnectionStageLabelFormatter.swift @@ -96,7 +96,7 @@ internal enum ConnectionStageLabelFormatter { return String(localized: "the SOCKS proxy") case .tunnelCommand: return String(localized: "the tunnel command") - case .remoteFile: + case .remoteFile, .remoteDatabaseSession: let host = connection.resolvedSSHConfig.host.trimmingCharacters(in: .whitespaces) return host.isEmpty ? nil : host case .none: diff --git a/TablePro/Models/Connection/ConnectionTunnelKind.swift b/TablePro/Models/Connection/ConnectionTunnelKind.swift index 1dccaf097b..dc8815c15d 100644 --- a/TablePro/Models/Connection/ConnectionTunnelKind.swift +++ b/TablePro/Models/Connection/ConnectionTunnelKind.swift @@ -19,6 +19,11 @@ enum ConnectionTunnelKind: String, CaseIterable, Sendable { /// that rebuilds it. What travels is a file rather than a socket. case remoteFile + /// A live database on an SSH server, reached by running statements on the server through the + /// SQLite agent over an exec channel exposed on a loopback port. Same SSH configuration as + /// `.remoteFile`, but the file is read and written in place rather than copied down. + case remoteDatabaseSession + /// The transports the connection form offers as switches the user can turn on independently. /// /// `.remoteFile` is deliberately absent. It is not a switch of its own: it is what an SSH @@ -36,6 +41,7 @@ enum ConnectionTunnelKind: String, CaseIterable, Sendable { case .socksProxy: return String(localized: "SOCKS Proxy") case .tunnelCommand: return String(localized: "Tunnel Command") case .remoteFile: return String(localized: "Remote Database File") + case .remoteDatabaseSession: return String(localized: "Remote Database File") } } @@ -58,6 +64,8 @@ enum ConnectionTunnelKind: String, CaseIterable, Sendable { return String(localized: "Holds a command that forwards a local port, such as kubectl port-forward.") case .remoteFile: return String(localized: "Copies a database file from an SSH server and opens the copy read-only.") + case .remoteDatabaseSession: + return String(localized: "Runs statements on the SSH server so a database file there can be read and written in place.") } } @@ -72,7 +80,7 @@ enum ConnectionTunnelKind: String, CaseIterable, Sendable { /// to the driver's own C library, inside the plugin. var carriesMeasuredBytes: Bool { switch self { - case .ssh, .socksProxy: return true + case .ssh, .socksProxy, .remoteDatabaseSession: return true case .cloudflare, .cloudSQLProxy, .tunnelCommand, .remoteFile: return false } } @@ -103,16 +111,38 @@ extension DatabaseConnection { /// main-actor isolated. `enabledTunnelKinds` is asked from wherever a connection is being /// resolved, so it cannot hop actors; the registry guards its own state with a lock for exactly /// this reason. + /// Whether this connection opens a read-only copy of a file on an SSH server. + /// + /// The copy is the fallback for any remote file that is not a live session: the read-only + /// choice, and also a `.onServer` choice on a type that has no agent to serve it. That keeps a + /// file-backed connection with a path from ever resolving to a plain SSH tunnel, which forwards + /// a port a file driver cannot use. var opensRemoteDatabaseFile: Bool { - guard resolvedSSHConfig.forwardsRemoteFile else { return false } + guard resolvedSSHConfig.forwardsRemoteFile, !opensRemoteDatabaseSession else { return false } guard let capabilities = PluginMetadataRegistry.shared.snapshot(for: type)?.capabilities else { return false } return capabilities.supportsRemoteDatabaseFile && capabilities.localFilePathField != nil } + /// Whether this connection runs its statements on the SSH server against a live database file. + /// + /// The same SSH configuration as the read-only copy, told apart by `remoteFileAccess`. A driver + /// whose type has no agent (everything but SQLite today) can never enter this mode, so a stored + /// `.onServer` on another type falls back to no transport rather than a session nothing serves. + var opensRemoteDatabaseSession: Bool { + guard resolvedSSHConfig.forwardsRemoteFile, resolvedSSHConfig.remoteFileAccess == .onServer else { + return false + } + guard let capabilities = PluginMetadataRegistry.shared.snapshot(for: type)?.capabilities + else { return false } + return capabilities.supportsRemoteDatabaseSession && capabilities.localFilePathField != nil + } + var enabledTunnelKinds: [ConnectionTunnelKind] { var kinds: [ConnectionTunnelKind] = [] - if opensRemoteDatabaseFile { + if opensRemoteDatabaseSession { + kinds.append(.remoteDatabaseSession) + } else if opensRemoteDatabaseFile { kinds.append(.remoteFile) } else if resolvedSSHConfig.enabled { kinds.append(.ssh) diff --git a/TablePro/Models/Connection/SSHTunnelFormState.swift b/TablePro/Models/Connection/SSHTunnelFormState.swift index d4bfd0a45a..f6c22dc8e1 100644 --- a/TablePro/Models/Connection/SSHTunnelFormState.swift +++ b/TablePro/Models/Connection/SSHTunnelFormState.swift @@ -40,6 +40,11 @@ struct SSHTunnelFormState { // Remote database file var remoteFilePath: String = "" + /// New connections default to running on the server, which is what a user reaching for a remote + /// SQLite database wants; loading an existing connection overwrites this with its saved value, + /// and a connection saved before the live mode existed decodes as the read-only copy. + var remoteFileAccess: RemoteFileAccess = .onServer + // MARK: - Computed Properties var selectedProfile: SSHProfile? { @@ -67,7 +72,8 @@ struct SSHTunnelFormState { totpAlgorithm: totpAlgorithm, totpDigits: totpDigits, totpPeriod: totpPeriod, - remoteFilePath: remoteFilePath + remoteFilePath: remoteFilePath, + remoteFileAccess: remoteFileAccess ) } @@ -80,6 +86,7 @@ struct SSHTunnelFormState { } var config = profile.toSSHConfiguration() config.remoteFilePath = remoteFilePath + config.remoteFileAccess = remoteFileAccess return config } @@ -95,11 +102,13 @@ struct SSHTunnelFormState { profileId = nil populateFields(from: config) remoteFilePath = config.remoteFilePath + remoteFileAccess = config.remoteFileAccess case .profile(let id, let snapshot): enabled = true profileId = id populateFields(from: snapshot) remoteFilePath = connection.sshConfig.remoteFilePath + remoteFileAccess = connection.sshConfig.remoteFileAccess } } @@ -124,6 +133,7 @@ struct SSHTunnelFormState { if let profileId, let profile = profiles.first(where: { $0.id == profileId }) { var snapshot = profile.toSSHConfiguration() snapshot.remoteFilePath = remoteFilePath + snapshot.remoteFileAccess = remoteFileAccess return .profile(id: profileId, snapshot: snapshot) } return .inline(buildInlineConfig()) diff --git a/TablePro/Models/Connection/SSHTypes.swift b/TablePro/Models/Connection/SSHTypes.swift index 4a33f239de..3a081c96fc 100644 --- a/TablePro/Models/Connection/SSHTypes.swift +++ b/TablePro/Models/Connection/SSHTypes.swift @@ -127,6 +127,16 @@ struct SSHJumpHost: Codable, Hashable, Identifiable { } } +/// How a file-backed connection reaches a database file that lives on an SSH server. +enum RemoteFileAccess: String, Codable, Hashable, Sendable { + /// Fetch a snapshot over SFTP and open the copy on this Mac, read-only. The original is never + /// written. This is what a connection saved before the live mode existed decodes to. + case readOnlyCopy + /// Run statements on the server against the live database through the SQLite agent, so reads + /// and writes act on the file the server's own programs are using. + case onServer +} + /// SSH tunnel configuration for database connections struct SSHConfiguration: Codable, Hashable { var enabled: Bool = false @@ -152,6 +162,11 @@ struct SSHConfiguration: Codable, Hashable { /// explainable. var remoteFilePath: String = "" + /// Whether the named file is opened as a read-only copy on this Mac or as a live session on the + /// server. Defaults to the copy, so a configuration written before the live mode existed keeps + /// its old behaviour, and an older app that cannot decode this key falls back the same way. + var remoteFileAccess: RemoteFileAccess = .readOnlyCopy + var forwardsRemoteFile: Bool { enabled && !remoteFilePath.isEmpty } /// Username may be empty: the runtime resolver supplies `User` from @@ -168,6 +183,7 @@ extension SSHConfiguration { case enabled, host, port, username, authMethod, privateKeyPath, agentSocketPath, jumpHosts case totpMode, totpAlgorithm, totpDigits, totpPeriod case remoteFilePath + case remoteFileAccess } /// Every property here declares a default, so every key decodes as optional. A required decode @@ -190,6 +206,7 @@ extension SSHConfiguration { totpDigits = try container.decodeIfPresent(Int.self, forKey: .totpDigits) ?? 6 totpPeriod = try container.decodeIfPresent(Int.self, forKey: .totpPeriod) ?? 30 remoteFilePath = try container.decodeIfPresent(String.self, forKey: .remoteFilePath) ?? "" + remoteFileAccess = try container.decodeIfPresent(RemoteFileAccess.self, forKey: .remoteFileAccess) ?? .readOnlyCopy } } diff --git a/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift b/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift index eb94f20444..c68c032512 100644 --- a/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift +++ b/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift @@ -10,7 +10,7 @@ extension ConnectionFormCoordinator { var safeModeFloor: SafeModeFloor? { SafeModeFloor.resolve( isEngineReadOnly: services.pluginManager.isEngineReadOnly(for: network.type), - opensRemoteDatabaseFile: transport == .remoteFile, + opensRemoteDatabaseFile: transport == .remoteFile && ssh.state.remoteFileAccess == .readOnlyCopy, managedMinimum: ManagedPolicyResolver.minimumSafeModeLevel(policy: ManagedPolicyReader.shared) ) } diff --git a/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+Transport.swift b/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+Transport.swift index 18485073c4..9168abc977 100644 --- a/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+Transport.swift +++ b/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+Transport.swift @@ -77,6 +77,13 @@ extension ConnectionFormCoordinator { services.pluginManager.supportsRemoteDatabaseFile(for: network.type) } + /// Whether the current type can run its statements on the server, which is what decides if the + /// Open pop-up appears beside the remote path. A type that only fetches a copy never shows it, + /// and its access stays the copy. + var supportsRemoteDatabaseSession: Bool { + services.pluginManager.supportsRemoteDatabaseSession(for: network.type) + } + var supportsSSL: Bool { services.pluginManager.supportsSSL(for: network.type) } diff --git a/TablePro/Views/ConnectionForm/Panes/NetworkPaneView.swift b/TablePro/Views/ConnectionForm/Panes/NetworkPaneView.swift index 5d5a52d1d0..8381cb23e6 100644 --- a/TablePro/Views/ConnectionForm/Panes/NetworkPaneView.swift +++ b/TablePro/Views/ConnectionForm/Panes/NetworkPaneView.swift @@ -63,7 +63,7 @@ struct NetworkPaneView: View { EmptyView() case .ssh: SSHTransportSections(coordinator: coordinator) - case .remoteFile: + case .remoteFile, .remoteDatabaseSession: RemoteFileTransportSections(coordinator: coordinator) case .cloudflare: CloudflareTransportSections(coordinator: coordinator) diff --git a/TablePro/Views/ConnectionForm/Panes/Transports/SSHTransportSections.swift b/TablePro/Views/ConnectionForm/Panes/Transports/SSHTransportSections.swift index ded60fe215..c3e7b6db4b 100644 --- a/TablePro/Views/ConnectionForm/Panes/Transports/SSHTransportSections.swift +++ b/TablePro/Views/ConnectionForm/Panes/Transports/SSHTransportSections.swift @@ -103,17 +103,33 @@ struct RemoteFileTransportSections: View { TextField(String(localized: "Path"), text: $coordinator.ssh.state.remoteFilePath) .autocorrectionDisabled() .accessibilityIdentifier("connection-form-remote-file-path") + if coordinator.supportsRemoteDatabaseSession { + Picker(String(localized: "Open"), selection: $coordinator.ssh.state.remoteFileAccess) { + Text(String(localized: "On the Server")).tag(RemoteFileAccess.onServer) + Text(String(localized: "As a Read-Only Copy")).tag(RemoteFileAccess.readOnlyCopy) + } + .accessibilityIdentifier("connection-form-remote-file-access") + } } header: { Text(String(localized: "Remote File")) } footer: { - VStack(alignment: .leading, spacing: 4) { - Text("Absolute, or relative to the SSH account's home directory. `~` works.") - Text("The file is copied to this Mac and opened read-only. The original on the server is never written to.") - } - .font(.caption) - .foregroundStyle(.secondary) + remoteFileFooter } SSHServerSections(sshState: $coordinator.ssh.state) } + + @ViewBuilder + private var remoteFileFooter: some View { + VStack(alignment: .leading, spacing: 4) { + Text("Absolute, or relative to the SSH account's home directory. `~` works.") + if coordinator.supportsRemoteDatabaseSession, coordinator.ssh.state.remoteFileAccess == .onServer { + Text("Statements run on the SSH server, so reads and writes act on the live database. The server needs python3.") + } else { + Text("The file is copied to this Mac and opened read-only. The original on the server is never written to.") + } + } + .font(.caption) + .foregroundStyle(.secondary) + } } diff --git a/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift b/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift new file mode 100644 index 0000000000..d33f1b313b --- /dev/null +++ b/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift @@ -0,0 +1,53 @@ +// +// RemoteSQLiteAdmissionTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct RemoteSQLiteAdmissionTests { + @Test func acceptsTheMatchingToken() { + let token = RemoteSQLiteAdmission.newToken() + let line = Data((RemoteSQLiteWire.admissionPrefix + token).utf8) + #expect(RemoteSQLiteAdmission.isAuthorized(line: line, token: token)) + } + + @Test func rejectsAWrongToken() { + let line = Data((RemoteSQLiteWire.admissionPrefix + "0000").utf8) + #expect(!RemoteSQLiteAdmission.isAuthorized(line: line, token: "1111")) + } + + @Test func rejectsAMissingPrefix() { + let line = Data("1111".utf8) + #expect(!RemoteSQLiteAdmission.isAuthorized(line: line, token: "1111")) + } + + @Test func rejectsAnEmptyTokenEvenWhenPresented() { + let line = Data(RemoteSQLiteWire.admissionPrefix.utf8) + #expect(!RemoteSQLiteAdmission.isAuthorized(line: line, token: "")) + } + + @Test func rejectsALineOverTheLengthCap() { + let token = String(repeating: "a", count: RemoteSQLiteAdmission.maxLineLength + 10) + let line = Data((RemoteSQLiteWire.admissionPrefix + token).utf8) + #expect(!RemoteSQLiteAdmission.isAuthorized(line: line, token: token)) + } + + @Test func newTokenIs64HexCharacters() { + let token = RemoteSQLiteAdmission.newToken() + #expect(token.count == 64) + #expect(token.allSatisfy { $0.isHexDigit }) + } + + @Test func launcherCommandCarriesNoUserDataAndFitsTheArgumentLimit() { + let command = RemoteSQLiteAgentSource.launcherCommand() + #expect(command.hasPrefix("sh -c '")) + #expect(command.contains(RemoteSQLiteWire.noPythonNotice)) + #expect(command.contains("base64.b64decode")) + // Well under the Linux MAX_ARG_STRLEN of 128 KiB. + #expect(command.utf8.count < 100_000) + } +} diff --git a/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift b/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift new file mode 100644 index 0000000000..0666e947dc --- /dev/null +++ b/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift @@ -0,0 +1,180 @@ +// +// RemoteSQLiteAgentPipeTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +/// Runs the SQLite agent that ships in the app against a real database, driven with the local +/// `python3` over pipes. It proves the embedded program and the plugin's codec agree on the wire, +/// that typed values and declared types come back the way the local driver reads them, and that the +/// server-side safety rules hold, without needing SSH. +struct RemoteSQLiteAgentPipeTests { + private static var python3Path: String? { + for candidate in ["/usr/bin/python3", "/opt/homebrew/bin/python3", "/usr/local/bin/python3"] where + FileManager.default.isExecutableFile(atPath: candidate) { + return candidate + } + return nil + } + + static var hasPython3: Bool { python3Path != nil } + + // MARK: - Harness + + private final class Agent { + private let process = Process() + private let toAgent = Pipe() + private let fromAgent = Pipe() + private var reader = SQLiteAgentFrameReader() + + init(pythonPath: String) throws { + let encoded = Data(RemoteSQLiteAgentSource.python.utf8).base64EncodedString() + process.executableURL = URL(fileURLWithPath: pythonPath) + process.arguments = ["-I", "-S", "-c", "import base64,sys;exec(base64.b64decode(sys.argv[1]))", encoded] + process.standardInput = toAgent + process.standardOutput = fromAgent + process.standardError = FileHandle.nullDevice + try process.run() + } + + func send(_ request: SQLiteAgentRequest) { + toAgent.fileHandleForWriting.write(SQLiteAgentFrameEncoder.encode(request)) + } + + func nextReply() throws -> SQLiteAgentReply { + while true { + if let reply = try reader.nextReply() { return reply } + let chunk = fromAgent.fileHandleForReading.availableData + if chunk.isEmpty { throw AgentError.closed } + reader.append(chunk) + } + } + + /// Reads a full EXEC response: header, then rows, then done or error. + func runExecute(_ sql: String, parameters: [SQLiteAgentValue] = []) throws -> ExecuteResult { + send(.execute(sql: sql, parameters: parameters, rowCap: 0)) + var columns: [SQLiteAgentColumn] = [] + var rows: [[SQLiteAgentValue]] = [] + while true { + switch try nextReply() { + case .header(let cols): columns = cols + case .rows(let count, let values): + var index = 0 + while index < values.count { + rows.append(Array(values[index..<(index + count)])) + index += count + } + case .done(let changes, let truncated): + return .ok(columns: columns, rows: rows, changes: changes, truncated: truncated) + case .error(_, let message): + return .error(message) + default: + throw AgentError.unexpected + } + } + } + + func close() { + toAgent.fileHandleForWriting.closeFile() + process.waitUntilExit() + } + + enum AgentError: Error { case closed, unexpected } + enum ExecuteResult { + case ok(columns: [SQLiteAgentColumn], rows: [[SQLiteAgentValue]], changes: Int64, truncated: Bool) + case error(String) + } + } + + private func withEmptyDatabase(_ body: (String, Agent) throws -> Void) throws { + let python = try #require(Self.python3Path) + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("agent-\(UUID().uuidString).db").path + FileManager.default.createFile(atPath: path, contents: Data()) + defer { try? FileManager.default.removeItem(atPath: path) } + + let agent = try Agent(pythonPath: python) + defer { agent.close() } + agent.send(.hello(protocolVersion: 1, path: path, busyTimeoutMilliseconds: 2000)) + guard case .ready = try agent.nextReply() else { + Issue.record("agent did not report ready") + return + } + try body(path, agent) + } + + // MARK: - Tests + + @Test(.enabled(if: RemoteSQLiteAgentPipeTests.hasPython3)) + func typedValuesAndDeclaredTypesMatchTheLocalDriver() throws { + try withEmptyDatabase { _, agent in + _ = try agent.runExecute("CREATE TABLE t(a INTEGER, b VARCHAR(9), c BLOB, d REAL)") + _ = try agent.runExecute( + "INSERT INTO t VALUES(?, ?, ?, ?)", + parameters: [.text(Data("9223372036854775807".utf8)), .text(Data("x".utf8)), .blob(Data([0x00, 0xFF])), .text(Data("0.1".utf8))] + ) + guard case .ok(let columns, let rows, _, _) = try agent.runExecute("SELECT a, b AS bee, c, d, a + 1 AS expr FROM t") else { + Issue.record("select failed") + return + } + #expect(columns.map(\.name) == ["a", "bee", "c", "d", "expr"]) + #expect(columns.map { $0.declaredType } == ["INTEGER", "VARCHAR(9)", "BLOB", "REAL", nil]) + #expect(rows.count == 1) + #expect(rows[0][0] == .text(Data("9223372036854775807".utf8))) + #expect(rows[0][2] == .blob(Data([0x00, 0xFF]))) + } + } + + @Test(.enabled(if: RemoteSQLiteAgentPipeTests.hasPython3)) + func insertReportsRowCount() throws { + try withEmptyDatabase { _, agent in + _ = try agent.runExecute("CREATE TABLE t(a INTEGER)") + guard case .ok(_, _, let changes, _) = try agent.runExecute("INSERT INTO t VALUES(1)") else { + Issue.record("insert failed") + return + } + #expect(changes == 1) + } + } + + @Test(.enabled(if: RemoteSQLiteAgentPipeTests.hasPython3)) + func fts3TokenizerIsDeniedByTheAuthorizer() throws { + try withEmptyDatabase { _, agent in + guard case .error(let message) = try agent.runExecute("SELECT fts3_tokenizer('evil', x'4141414141414141')") else { + Issue.record("fts3_tokenizer was not denied") + return + } + #expect(message.contains("not authorized") || message.contains("no such function")) + } + } + + @Test(.enabled(if: RemoteSQLiteAgentPipeTests.hasPython3)) + func loadExtensionIsUnavailable() throws { + try withEmptyDatabase { _, agent in + guard case .error = try agent.runExecute("SELECT load_extension('x')") else { + Issue.record("load_extension was not blocked") + return + } + } + } + + @Test(.enabled(if: RemoteSQLiteAgentPipeTests.hasPython3)) + func openingAMissingFileFailsWithoutCreatingIt() throws { + let python = try #require(Self.python3Path) + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("missing-\(UUID().uuidString).db").path + let agent = try Agent(pythonPath: python) + defer { agent.close() } + agent.send(.hello(protocolVersion: 1, path: path, busyTimeoutMilliseconds: 2000)) + guard case .failure(let failure) = try agent.nextReply() else { + Issue.record("expected a failure for a missing file") + return + } + #expect(failure.code == SQLiteAgentFailure.openFailed) + #expect(!FileManager.default.fileExists(atPath: path)) + } +} diff --git a/TableProTests/RemoteSQLite/RemoteSQLiteModelTests.swift b/TableProTests/RemoteSQLite/RemoteSQLiteModelTests.swift new file mode 100644 index 0000000000..b3ea000497 --- /dev/null +++ b/TableProTests/RemoteSQLite/RemoteSQLiteModelTests.swift @@ -0,0 +1,92 @@ +// +// RemoteSQLiteModelTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct RemoteSQLiteModelTests { + private func sqliteConnection(access: RemoteFileAccess, path: String = "/srv/app.db") -> DatabaseConnection { + var ssh = SSHConfiguration() + ssh.enabled = true + ssh.host = "prod-1" + ssh.username = "deploy" + ssh.remoteFilePath = path + ssh.remoteFileAccess = access + return DatabaseConnection(name: "remote", type: .sqlite, sshTunnelMode: .inline(ssh)) + } + + @Test func absentAccessDecodesAsReadOnlyCopy() throws { + let json = Data(""" + {"enabled":true,"host":"prod-1","username":"deploy","remoteFilePath":"/srv/app.db"} + """.utf8) + let config = try JSONDecoder().decode(SSHConfiguration.self, from: json) + #expect(config.remoteFileAccess == .readOnlyCopy) + } + + @Test func accessRoundTripsThroughCoding() throws { + var config = SSHConfiguration() + config.enabled = true + config.remoteFilePath = "/srv/app.db" + config.remoteFileAccess = .onServer + let data = try JSONEncoder().encode(config) + let decoded = try JSONDecoder().decode(SSHConfiguration.self, from: data) + #expect(decoded.remoteFileAccess == .onServer) + } + + @Test func onServerResolvesToLiveSessionForSQLite() { + let connection = sqliteConnection(access: .onServer) + #expect(connection.opensRemoteDatabaseSession) + #expect(!connection.opensRemoteDatabaseFile) + #expect(connection.enabledTunnelKinds == [.remoteDatabaseSession]) + #expect(connection.activeTunnelKind == .remoteDatabaseSession) + } + + @Test func readOnlyCopyResolvesToCopyForSQLite() { + let connection = sqliteConnection(access: .readOnlyCopy) + #expect(!connection.opensRemoteDatabaseSession) + #expect(connection.opensRemoteDatabaseFile) + #expect(connection.enabledTunnelKinds == [.remoteFile]) + } + + @Test func aFileBackedConnectionWithAPathNeverResolvesToPlainSSH() { + // Both modes are a remote file, never a port forward, whatever the access says. + #expect(sqliteConnection(access: .onServer).enabledTunnelKinds != [.ssh]) + #expect(sqliteConnection(access: .readOnlyCopy).enabledTunnelKinds != [.ssh]) + } + + @Test func liveSessionCarriesNoReadOnlyFloor() { + let floor = SafeModeFloor.resolve( + isEngineReadOnly: false, + opensRemoteDatabaseFile: sqliteConnection(access: .onServer).opensRemoteDatabaseFile, + managedMinimum: nil + ) + #expect(floor == nil) + } + + @Test func readOnlyCopyKeepsTheReadOnlyFloor() { + let floor = SafeModeFloor.resolve( + isEngineReadOnly: false, + opensRemoteDatabaseFile: sqliteConnection(access: .readOnlyCopy).opensRemoteDatabaseFile, + managedMinimum: nil + ) + #expect(floor?.level == .readOnly) + #expect(floor?.reason == .remoteDatabaseFile) + } + + @Test func effectiveConnectionCarriesBackendMarkerTokenAndEndpoint() { + let connection = sqliteConnection(access: .onServer) + let effective = connection.openingRemoteSQLiteSession( + host: "127.0.0.1", port: 61234, token: "deadbeef", remotePath: "/srv/app.db", field: .database + ) + #expect(effective.host == "127.0.0.1") + #expect(effective.port == 61234) + #expect(effective.database == "/srv/app.db") + #expect(effective.additionalFields[RemoteSQLiteWire.backendFieldKey] == RemoteSQLiteWire.agentBackendValue) + #expect(effective.additionalFields[RemoteSQLiteWire.tokenFieldKey] == "deadbeef") + #expect(!effective.resolvedSSHConfig.enabled) + } +} diff --git a/TableProTests/RemoteSQLite/RemoteSQLiteWireParityTests.swift b/TableProTests/RemoteSQLite/RemoteSQLiteWireParityTests.swift new file mode 100644 index 0000000000..ab29029a1a --- /dev/null +++ b/TableProTests/RemoteSQLite/RemoteSQLiteWireParityTests.swift @@ -0,0 +1,30 @@ +// +// RemoteSQLiteWireParityTests.swift +// TableProTests +// + +import Testing + +@testable import TablePro + +/// The app builds the launcher and admits clients; the SQLite plugin runs the codec. They share a +/// handful of wire constants that nothing at compile time forces to agree, because the plugin can +/// ship as a separate module. This reads both sides and fails on any drift. +struct RemoteSQLiteWireParityTests { + @Test func appAndPluginAgreeOnWireConstants() { + #expect(RemoteSQLiteWire.backendFieldKey == SQLiteAgentProtocol.backendFieldKey) + #expect(RemoteSQLiteWire.agentBackendValue == SQLiteAgentProtocol.agentBackendValue) + #expect(RemoteSQLiteWire.tokenFieldKey == SQLiteAgentProtocol.tokenFieldKey) + #expect(RemoteSQLiteWire.admissionPrefix == SQLiteAgentProtocol.admissionPrefix) + #expect(RemoteSQLiteWire.noPythonNotice == SQLiteAgentProtocol.noPythonNotice) + } + + @Test func noPythonNoticeIsRecognizedAsALauncherNotice() { + #expect(RemoteSQLiteWire.noPythonNotice.hasPrefix(SQLiteAgentProtocol.launcherNoticePrefix)) + } + + @Test func admissionLinesMatchOnBothSides() { + let token = "abc123" + #expect(RemoteSQLiteWire.admissionLine(token: token) == SQLiteAgentProtocol.admissionPreamble(token: token)) + } +} diff --git a/TableProTests/RemoteSQLite/SQLiteAgentProtocolTests.swift b/TableProTests/RemoteSQLite/SQLiteAgentProtocolTests.swift new file mode 100644 index 0000000000..626e1cfc00 --- /dev/null +++ b/TableProTests/RemoteSQLite/SQLiteAgentProtocolTests.swift @@ -0,0 +1,97 @@ +// +// SQLiteAgentProtocolTests.swift +// TableProTests +// + +import Foundation +import Testing + +struct SQLiteAgentProtocolTests { + private func roundTrip(_ reply: SQLiteAgentReply) throws -> SQLiteAgentReply { + var reader = SQLiteAgentFrameReader() + reader.append(SQLiteAgentFrameEncoder.encode(reply)) + let decoded = try reader.nextReply() + return try #require(decoded) + } + + private func roundTrip(_ request: SQLiteAgentRequest) throws -> SQLiteAgentRequest { + var reader = SQLiteAgentFrameReader() + reader.append(SQLiteAgentFrameEncoder.encode(request)) + let decoded = try reader.nextRequest() + return try #require(decoded) + } + + @Test func readyRoundTrips() throws { + let reply = try roundTrip(.ready(protocolVersion: 1, sqliteVersion: "3.45.1", pythonVersion: "3.12.3")) + #expect(reply == .ready(protocolVersion: 1, sqliteVersion: "3.45.1", pythonVersion: "3.12.3")) + } + + @Test func headerCarriesNilAndPresentDeclaredTypes() throws { + let columns = [ + SQLiteAgentColumn(name: "a", declaredType: "INTEGER"), + SQLiteAgentColumn(name: "expr", declaredType: nil), + SQLiteAgentColumn(name: "b", declaredType: "VARCHAR(9)"), + ] + let reply = try roundTrip(.header(columns)) + #expect(reply == .header(columns)) + } + + @Test func rowsPreserveNullEmptyTextAndBlobBytes() throws { + let values: [SQLiteAgentValue] = [ + .null, + .text(Data()), + .text(Data("9223372036854775807".utf8)), + .blob(Data([0x00, 0xFF])), + ] + let reply = try roundTrip(.rows(columnCount: 2, values: values)) + #expect(reply == .rows(columnCount: 2, values: values)) + } + + @Test func doneCarriesNegativeAndLargeChanges() throws { + #expect(try roundTrip(.done(changes: 0, truncated: false)) == .done(changes: 0, truncated: false)) + #expect(try roundTrip(.done(changes: 5, truncated: true)) == .done(changes: 5, truncated: true)) + } + + @Test func errorRoundTripsWithNegativeCode() throws { + let reply = try roundTrip(.error(code: -1, message: "not authorized")) + #expect(reply == .error(code: -1, message: "not authorized")) + } + + @Test func executeRequestRoundTripsWithMixedParameters() throws { + let request = SQLiteAgentRequest.execute( + sql: "SELECT * FROM t WHERE a = ? AND b = ?", + parameters: [.text(Data("x".utf8)), .blob(Data([0x01, 0x02]))], + rowCap: 500 + ) + #expect(try roundTrip(request) == request) + } + + @Test func helloRoundTrips() throws { + let request = SQLiteAgentRequest.hello(protocolVersion: 1, path: "~/app.db", busyTimeoutMilliseconds: 2000) + #expect(try roundTrip(request) == request) + } + + @Test func launcherNoticeIsReadBeforeAnyFrame() throws { + var reader = SQLiteAgentFrameReader() + reader.append(Data((SQLiteAgentProtocol.noPythonNotice + "\n").utf8)) + let notice = try reader.nextReply() + #expect(notice == .launcherNotice(SQLiteAgentProtocol.noPythonNotice)) + } + + @Test func framesReassembleAcrossChunkBoundaries() throws { + let encoded = SQLiteAgentFrameEncoder.encode(SQLiteAgentReply.done(changes: 3, truncated: false)) + var reader = SQLiteAgentFrameReader() + reader.append(encoded.prefix(2)) + #expect(try reader.nextReply() == nil) + reader.append(encoded.suffix(from: encoded.index(encoded.startIndex, offsetBy: 2))) + #expect(try reader.nextReply() == .done(changes: 3, truncated: false)) + } + + @Test func oversizeFrameLengthIsRejected() { + var reader = SQLiteAgentFrameReader() + var header = Data() + withUnsafeBytes(of: UInt32(SQLiteAgentProtocol.maxFrameLength + 1).bigEndian) { header.append(contentsOf: $0) } + reader.append(header) + #expect(throws: SQLiteAgentProtocolError.self) { _ = try reader.nextReply() } + } +} diff --git a/docs/connections/remote-database-files.mdx b/docs/connections/remote-database-files.mdx index cd6c7d00c2..82e7503758 100644 --- a/docs/connections/remote-database-files.mdx +++ b/docs/connections/remote-database-files.mdx @@ -1,11 +1,11 @@ --- title: Remote Database Files -description: Open a SQLite database that lives on an SSH server, working from a read-only copy fetched over SFTP +description: Work with a SQLite database on an SSH server, live on the server or as a read-only copy on this Mac --- -Nothing is tunnelled here, and nothing is written. The database is fetched over SFTP, opened from the copy on this Mac, and the original on the server is never touched. +A SQLite database on an SSH server can be reached two ways, and the **Open** menu beside the path is where you choose. Run your statements on the server against the live file, or fetch a read-only copy and open it here. -That constraint is the feature, not a gap in it. SQLite's own documentation says its file locking cannot be relied on across a network, so a client that wrote to the file in place would eventually corrupt it. +The choice turns on where SQLite runs. Its file locking is only reliable for a process on the same machine as the file, so a client that wrote across the network would eventually corrupt the database. Running on the server keeps every read and write on that one machine; the copy sidesteps the question by never writing at all. ## Set one up @@ -17,7 +17,10 @@ That constraint is the feature, not a gap in it. SQLite's own documentation says Fill in **SSH Host**, **SSH Port**, and **SSH User**, then pick an authentication method. Password, private key, SSH agent, keyboard-interactive, jump hosts, and one-time codes all work the way they do for a tunnel, and a saved [SSH profile](/connections/ssh-profiles) supplies all of it at once. - Put the database's path on the server in **Path**. An absolute path is taken as written. A relative path, or one starting `~`, resolves against the SSH account's home directory, and a symlink resolves to the file it names. + Put the database's path on the server in **Path**. An absolute path is taken as written. A relative path, or one starting `~`, resolves against the SSH account's home directory. + + + Set **Open** to **On the Server** for live reads and writes, or **As a Read-Only Copy** to browse a snapshot on this Mac. New connections start on the server. @@ -25,39 +28,43 @@ That constraint is the feature, not a gap in it. SQLite's own documentation says The connection list shows the origin as `deploy@prod-1:/srv/app.db` instead of a local path, so a remote database never reads like one of your own. -## What arrives +## On the Server -Where the server carries `sqlite3` 3.27 or newer, it is asked for a `VACUUM INTO` snapshot and that is what comes down. This is the case worth having: the snapshot is taken while other programs keep writing and is still internally consistent. Measured against a database taking 22,518 committed transactions during the copy, the snapshot passed `PRAGMA integrity_check`. +Your statements run on the host, so the sidebar, the grid, the SQL editor and every structure edit act on the live database, alongside whatever else is writing it. Nothing is copied and nothing is installed: TablePro sends a small Python program over the SSH connection, and that program opens the file with the server's own SQLite. -Otherwise the file is copied directly, along with any `-wal` or `-journal` beside it. Those are not optional. A database in WAL mode keeps committed rows in `-wal` until a checkpoint moves them, so the main file alone is short of the most recent work with nothing to say so. +The server needs `python3`, version 3.6 or newer, with its standard library. Ubuntu, Debian and RHEL cloud images ship it; a stripped container may not. When it is missing, the connection fails with a message that says so, and **As a Read-Only Copy** is the fallback. -A direct copy of a database another program is writing at that moment can hold a mix of old and new pages. Take one when nothing else is writing, or install `sqlite3` on the server so the snapshot path is available. +Statements run on the server as your SSH user, against the live database. A write commits in place. Leave [Safe Mode](/features/safe-mode) at **Read-Only** on connections you only mean to browse. -What arrives is checked before anything opens it: the byte count has to match what the server reported, the file has to start like a SQLite database, and it has to pass `integrity_check`. A transfer cut short by a dropped session leaves an intact header and a plausible prefix, and `sqlite3_open` on that reports success and shows an empty database. +The server's SQLite is the one that runs the query, so its version sets what the editor and the structure tools can do. A `DROP COLUMN` needs 3.35, and editing a `CHECK` constraint through `ALTER TABLE` needs 3.53; on an older server the statement is refused with the server's own error. `SELECT version()` in the editor reports what the server runs. + +## As a Read-Only Copy + +The database is fetched over SFTP, opened from the copy on this Mac, and the original on the server is never written to. The connection runs at Safe Mode [**Read-Only**](/features/safe-mode#connections-that-are-always-read-only): editing a cell, running an `INSERT`, and altering a table are all refused, because an edit would land in the copy and disappear the next time the file was fetched. + +Where the server carries `sqlite3` 3.27 or newer, it is asked for a `VACUUM INTO` snapshot and that is what comes down. This is the case worth having: the snapshot is taken while other programs keep writing and is still internally consistent. Measured against a database taking 22,518 committed transactions during the copy, the snapshot passed `PRAGMA integrity_check`. -## Read-only, and what that means +Otherwise the file is copied directly, along with any `-wal` or `-journal` beside it. Those are not optional. A database in WAL mode keeps committed rows in `-wal` until a checkpoint moves them, so the main file alone is short of the most recent work. -The connection opens in read-only mode, the same mode the **Safe Mode** setting offers. Editing a cell, running an `INSERT`, and altering a table are all refused, by the grid and by the SQL editor alike. + +A direct copy of a database another program is writing at that moment can hold a mix of old and new pages. Take one when nothing else is writing, install `sqlite3` on the server so the snapshot path is available, or open the database on the server instead. + -That is deliberate rather than unfinished. An edit would land in the copy on this Mac, change nothing on the server, and disappear the next time the file was fetched. +What arrives is checked before anything opens it: the byte count has to match what the server reported, the file has to start like a SQLite database, and it has to pass `integrity_check`. A transfer cut short by a dropped session leaves an intact header and a plausible prefix, and `sqlite3_open` on that reports success and shows an empty database. ## Reconnecting -Closing a connection leaves its copy on disk. Opening it again compares the file on the server against what was fetched: if neither its size, its modification time, nor its write-ahead log has moved, the copy is reused and nothing is transferred. This matters more than it sounds, because the health monitor reconnects on a failed thirty-second ping, and a large database would otherwise come down again on every network blip. +Closing a connection ends the server session, or leaves the copy on disk. Opening it again reconnects, or compares the file on the server against what was fetched and reuses the copy when neither its size, its modification time, nor its write-ahead log has moved. The health check watches a live server session and reconnects it after a dropped connection, the same way an SSH tunnel recovers. ## Limitations No remote file browser. Type the path. -No writing back. A change made locally is refused before it can be made, so there is nothing to send. - SQLite only. DuckDB and libSQL open local files too, but their recovery-log naming and their local-versus-remote modes need handling this does not yet have. -Two connections naming one remote file share a single local copy and take turns fetching it. Two copies of TablePro on two Macs cannot see each other at all. - -Cancelling a fetch stops it between chunks rather than mid-request, so a transfer already inside a read finishes that read first. +Two connections naming one remote file each open their own session or copy. Two copies of TablePro on two Macs cannot see each other; the server's own locking keeps their writes apart. ## Related diff --git a/docs/databases/sqlite.mdx b/docs/databases/sqlite.mdx index cc2ede636a..d03546b620 100644 --- a/docs/databases/sqlite.mdx +++ b/docs/databases/sqlite.mdx @@ -53,9 +53,9 @@ One connection is one file, with no database to switch between, and the object l ## A database on another machine -The **Remote File** pane points the connection at a database on an SSH server. It is fetched over SFTP and opened from a copy on this Mac, and the original is never written to. The connection runs at Safe Mode [**Read-Only**](/features/safe-mode#connections-that-are-always-read-only), so the grid and the editor refuse edits that would only change the copy. +The **Remote File** pane points the connection at a database on an SSH server, and the **Open** menu beside the path decides how it is reached. **On the Server** runs your statements on the server against the live file, so reads and writes land on the database the server's own programs are using. **As a Read-Only Copy** fetches a snapshot over SFTP and opens it on this Mac, and the original is never touched. -Where the server has `sqlite3` 3.27 or newer, the copy is a `VACUUM INTO` snapshot, which stays consistent even while other programs write to the database. See [Remote Database Files](/connections/remote-database-files). +On the Server needs `python3` on the host, which every mainstream cloud image already carries, and nothing is installed. The read-only copy needs only SFTP, and it runs at Safe Mode [**Read-Only**](/features/safe-mode#connections-that-are-always-read-only). See [Remote Database Files](/connections/remote-database-files). ## Edits that recreate the table diff --git a/project.yml b/project.yml index 9b8ad294b0..49a06faf4c 100644 --- a/project.yml +++ b/project.yml @@ -508,6 +508,7 @@ targets: - Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift - Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift - Plugins/SQLiteDriverPlugin/SQLiteMaintenance.swift + - Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift - Plugins/LibSQLDriverPlugin/LibSQLDefaultValue.swift - Plugins/LibSQLDriverPlugin/HranaHttpClient.swift - Plugins/CloudflareD1DriverPlugin/CloudflareD1DefaultValue.swift