ssl: delay closing TCP sockets in start_server

Some tests expect server-side SSLSocket#accept to fail for various
reasons. On some systems, closing the underlying socket immediately
with IO#close causes the TCP connection to be terminated with RST.

Do not close it immediately so that the client can reliably receive
the TLS alert. This allows writing more meaningful assertions.

Also add a dedicated test case for the rb_sys_fail() path in
SSLSocket#connect.

Author: rhenium

test/openssl/test_ssl.rb

@@ -491,7 +491,8 @@ def test_verify_mode_client_cert_required
     sctx.verify_mode =
       OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
     start_server(sctx, ignore_listener_error: true) { |port|
-      assert_handshake_error {
+      # TLS 1.3 alert: certificate_required(116)
+      assert_raise_with_message(OpenSSL::SSL::SSLError, /alert number 116/) {
         server_connect(port) { |ssl| ssl.puts("abc"); ssl.gets }
       }
     }
@@ -547,7 +548,7 @@ def test_client_cert_cb_ignore_error
       # 1. Exception in client_cert_cb is suppressed
       # 2. No client certificate will be sent to the server
       # 3. SSL_VERIFY_FAIL_IF_NO_PEER_CERT causes the handshake to fail
-      assert_handshake_error {
+      assert_raise(OpenSSL::SSL::SSLError) {
         server_connect(port, ctx) { |ssl| ssl.puts("abc"); ssl.gets }
       }
     end
@@ -1286,6 +1287,27 @@ def test_verify_hostname_failure_error_code
     end
   end
 
+  def test_connect_systemcallerror
+    # SSL_connect() should fail with SSL_ERROR_SYSCALL and errno should be
+    # kept intact from the underlying recv(2)/send(2).
+    pend "AWS-LC does not preserve errno on SSL_ERROR_SYSCALL" if aws_lc?
+
+    server_proc = proc do |sock|
+      sock.setsockopt(:SOCKET, :LINGER, [1, 0].pack("ii"))
+      sock.read(1)
+      sock.close
+    end
+    start_server_proc(server_proc) do |port|
+      sock = TCPSocket.new("127.0.0.1", port)
+      ssl = OpenSSL::SSL::SSLSocket.new(sock)
+      assert_raise(Errno::ECONNRESET, Errno::EPIPE) {
+        ssl.connect
+      }
+    ensure
+      sock&.close
+    end
+  end
+
   def test_connect_certificate_verify_failed_exception_message
     start_server(ignore_listener_error: true) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
@@ -1338,7 +1360,7 @@ def check_supported_protocol_versions
           ssl.puts "abc"; assert_equal "abc\n", ssl.gets
         }
         supported << ver
-      rescue OpenSSL::SSL::SSLError, Errno::ECONNRESET
+      rescue OpenSSL::SSL::SSLError
       end
     end
 
@@ -2106,7 +2128,7 @@ def test_client_sigalgs
     start_server(sctx, ignore_listener_error: true) do |port|
       ctx1 = OpenSSL::SSL::SSLContext.new
       ctx1.add_certificate(@cli_cert, @cli_key) # RSA
-      assert_handshake_error {
+      assert_raise(OpenSSL::SSL::SSLError) {
         server_connect(port, ctx1) { |ssl|
           ssl.puts("abc"); ssl.gets
         }
@@ -2458,14 +2480,6 @@ def server_connect(port, ctx = nil)
       sock.close
     end
   end
-
-  def assert_handshake_error
-    # different OpenSSL versions react differently when facing a SSL/TLS version
-    # that has been marked as forbidden, therefore any of these may be raised
-    assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::EPIPE) {
-      yield
-    }
-  end
 end
 
 end

test/openssl/utils.rb

@@ -210,20 +210,19 @@ def start_server_proc(server_proc, &block)
       port = tcps.connect_address.ip_port
 
       threads = []
+      sockets = []
       server_thread = Thread.new do
         Thread.current.report_on_exception = false
 
         loop do
           readable, = IO.select([tcps, stop_pipe_r])
           break if readable.include? stop_pipe_r
-          sock = tcps.accept
+          sockets << sock = tcps.accept
 
           th = Thread.new do
             Thread.current.report_on_exception = false
 
             server_proc.call(sock)
-          ensure
-            sock.close
           end
           threads << th
         end
@@ -254,6 +253,7 @@ def start_server_proc(server_proc, &block)
         rescue Exception
         end
       }
+      sockets.each(&:close)
       raise pend if pend
       assert_join_threads(threads)
     end