Add support for custom BIO method

ext/openssl/extconf.rb

@@ -146,6 +146,8 @@ def find_openssl_library
 
 # added in 1.1.0, currently not in LibreSSL
 have_func("EVP_PBE_scrypt(\"\", 0, (unsigned char *)\"\", 0, 0, 0, 0, 0, NULL, 0)", evp_h)
+have_func("BIO_meth_new");
+have_func("SSL_set0_rbio");
 
 # added in OpenSSL 1.1.1 and LibreSSL 3.5.0, then removed in LibreSSL 4.0.0
 have_func("EVP_PKEY_check(NULL)", evp_h)
@@ -169,6 +171,9 @@ def find_openssl_library
 # added in 3.5.0
 have_func("SSL_get0_peer_signature_name(NULL, NULL)", ssl_h)
 
+have_func("rb_io_buffer_new")
+have_func("rb_io_buffer_free_locked")
+
 Logging::message "=== Checking done. ===\n"
 
 # Append flags from environment variables.

ext/openssl/ossl.h

@@ -78,6 +78,10 @@
 # define OSSL_HAVE_IMMUTABLE_PKEY
 #endif
 
+#if defined(HAVE_BIO_METH_NEW) && defined(HAVE_RB_IO_BUFFER_NEW) && defined(HAVE_RB_IO_BUFFER_FREE_LOCKED)
+# define OSSL_CUSTOM_BIO
+#endif
+
 /*
  * Common Module
  */

ext/openssl/ossl_ssl.c

@@ -10,6 +10,7 @@
  * (See the file 'COPYING'.)
  */
 #include "ossl.h"
+#include "ossl_ssl_custom_bio.h"
 
 #ifndef OPENSSL_NO_SOCK
 #define numberof(ary) (int)(sizeof(ary)/sizeof((ary)[0]))
@@ -29,7 +30,7 @@
 } while (0)
 
 VALUE mSSL;
-static VALUE eSSLError;
+VALUE eSSLError;
 static VALUE cSSLContext;
 VALUE cSSLSocket;
 
@@ -48,6 +49,7 @@ static ID id_i_cert_store, id_i_ca_file, id_i_ca_path, id_i_verify_mode,
           id_i_alpn_select_cb, id_i_alpn_protocols, id_i_servername_cb,
           id_i_verify_hostname, id_i_keylog_cb, id_i_tmp_dh_callback;
 static ID id_i_io, id_i_context, id_i_hostname;
+static ID id_i_bio_method;
 
 static int ossl_ssl_ex_ptr_idx;
 static int ossl_sslctx_ex_ptr_idx;
@@ -2700,6 +2702,92 @@ ossl_ssl_get_group(VALUE self)
 
 #endif /* !defined(OPENSSL_NO_SOCK) */
 
+#ifdef OSSL_CUSTOM_BIO
+/*
+ * call-seq:
+ *    ssl.bio_method => method or nil
+ *
+ * Returns the BIO method for the socket, or nil if not set. See also
+ * SSLSocket#bio_method=.
+ */
+static VALUE
+ossl_ssl_get_bio_method(VALUE self)
+{
+    return rb_ivar_get(self, id_i_bio_method);
+}
+
+/*
+ * call-seq:
+ *    ssl.bio_method = nil
+ *    ssl.bio_method = io
+ *    ssl.bio_method = [->(buf, maxlen) { ... }, ->(buf, len) { ... }]
+ *
+ * Sets the BIO method for the SSL socket. By default, the SSL connection uses a
+ * socket BIO for performing I/O, which means that OpenSSL will bypass the
+ * I/O implementation in the standard library, only using it for checking for
+ * I/O readiness.
+ *
+ * When the BIO method is set to an IO instance (normally the underlying socket
+ * instance), OpenSSL will read and write to the connection by calling the
+ * `#read` and `#write` methods on the given IO instance. This also allows
+ * better integration with a fiber scheduler for applications that use
+ * fiber-based concurrency.
+ *
+ * Alternatively, the BIO method may be customized by setting it to an array
+ * containing a read proc and a write proc. The read proc takes as parameters an
+ * IO::Buffer and the maximum number of bytes to read. The proc should return
+ * the number of bytes read. The write proc takes as parameters an IO::Buffer
+ * and the number of bytes to write. It should return the number of bytes
+ * written. Example usage:
+ *
+ *     io = ssl.to_io
+ *     ssl.bio_method = [
+ *       ->(buf, maxlen) {
+ *         str = io.read(maxlen)
+ *         len = str.bytesize
+ *         buf.set_string(str)
+ *         len
+ *       },
+ *       ->(buf, len) {
+ *         str = buf.get_string(0, len)
+ *         io.write(str)
+ *       }
+ *     ]
+ */
+static VALUE
+ossl_ssl_set_bio_method(VALUE self, VALUE method)
+{
+    SSL *ssl;
+    GetSSL(self, ssl);
+
+    switch(TYPE(method)) {
+        case T_FILE:
+        case T_OBJECT:
+        case T_STRUCT:
+            break;
+        case T_ARRAY:
+            if (RARRAY_LEN(method) != 2)
+                rb_raise(eSSLError, "Invalid BIO method");
+            break;
+        default:
+            rb_raise(eSSLError, "Invalid BIO method");
+    }
+
+    rb_ivar_set(self, id_i_bio_method, method);
+
+    if (NIL_P(method)) {
+        VALUE io = rb_ivar_get(self, id_i_io);
+        if (!SSL_set_fd(ssl, TO_SOCKET(rb_io_descriptor(io))))
+            ossl_raise(eSSLError, "SSL_set_fd");
+    }
+    else {
+        ossl_ssl_set_custom_bio(ssl, method);
+    }
+
+    return self;
+}
+#endif
+
 void
 Init_ossl_ssl(void)
 {
@@ -3133,6 +3221,10 @@ Init_ossl_ssl(void)
     rb_define_method(cSSLSocket, "tmp_key", ossl_ssl_tmp_key, 0);
     rb_define_method(cSSLSocket, "alpn_protocol", ossl_ssl_alpn_protocol, 0);
     rb_define_method(cSSLSocket, "export_keying_material", ossl_ssl_export_keying_material, -1);
+#ifdef OSSL_CUSTOM_BIO
+    rb_define_method(cSSLSocket, "bio_method", ossl_ssl_get_bio_method, 0);
+    rb_define_method(cSSLSocket, "bio_method=", ossl_ssl_set_bio_method, 1);
+#endif
 # ifdef OSSL_USE_NEXTPROTONEG
     rb_define_method(cSSLSocket, "npn_protocol", ossl_ssl_npn_protocol, 0);
 # endif
@@ -3300,5 +3392,6 @@ Init_ossl_ssl(void)
     DefIVarID(io);
     DefIVarID(context);
     DefIVarID(hostname);
+    DefIVarID(bio_method);
 #endif /* !defined(OPENSSL_NO_SOCK) */
 }

ext/openssl/ossl_ssl_custom_bio.c

@@ -0,0 +1,144 @@
+/*
+ * 'OpenSSL for Ruby' project
+ * Copyright (C) 2026  Sharon Rosner <sharon@noteflakes.com>
+ * All rights reserved.
+ */
+/*
+ * This program is licensed under the same licence as Ruby.
+ * (See the file 'COPYING'.)
+*/
+#include "ossl.h"
+
+#ifdef OSSL_CUSTOM_BIO
+
+#include "ossl_ssl_custom_bio.h"
+#include "ruby/io/buffer.h"
+
+extern VALUE eSSLError;
+static ID id_read, id_write, id_call, id_eof_p;
+
+static int
+ossl_ssl_custom_bio_in_read(BIO *bio, char *buf, int blen)
+{
+    VALUE target = (VALUE)BIO_get_data(bio);
+
+    switch(TYPE(target)) {
+        case T_FILE:
+        case T_OBJECT:
+        case T_STRUCT: {
+            VALUE str = rb_funcall(target, id_read, 1, INT2NUM(blen));
+            long slen = RSTRING_LEN(str);
+            memcpy(buf, RSTRING_PTR(str), slen);
+            RB_GC_GUARD(str);
+            return (int)slen;
+        }
+        case T_ARRAY: {
+            VALUE read_proc = rb_ary_entry(target, 0);
+            VALUE buffer = rb_io_buffer_new(buf, blen, RB_IO_BUFFER_LOCKED);
+            VALUE res = rb_funcall(read_proc, id_call, 2, buffer, INT2NUM(blen));
+            rb_io_buffer_free_locked(buffer);
+            return NUM2INT(res);
+        }
+        default:
+            rb_raise(eSSLError, "Invalid BIO target");
+    }
+}
+
+static int
+ossl_ssl_custom_bio_out_write(BIO *bio, const char *buf, int blen)
+{
+    VALUE target = (VALUE)BIO_get_data(bio);
+    switch(TYPE(target)) {
+        case T_FILE:
+        case T_OBJECT:
+        case T_STRUCT: {
+            VALUE str = rb_str_new(buf, blen);
+            VALUE res = rb_funcall(target, id_write, 1, str);
+            RB_GC_GUARD(str);
+            return NUM2INT(res);
+        }
+        case T_ARRAY: {
+            VALUE write_proc = rb_ary_entry(target, 1);
+            VALUE buffer = rb_io_buffer_new((char *)buf, blen, RB_IO_BUFFER_LOCKED | RB_IO_BUFFER_READONLY);
+            VALUE res = rb_funcall(write_proc, id_call, 2, buffer, INT2NUM(blen));
+            RB_GC_GUARD(buffer);
+            rb_io_buffer_free_locked(buffer);
+            return NUM2INT(res);
+        }
+        default:
+            rb_raise(eSSLError, "Invalid BIO target");
+    }
+}
+
+static long
+ossl_ssl_custom_bio_ctrl(BIO *bio, int cmd, long num, void *ptr)
+{
+    VALUE target = (VALUE)BIO_get_data(bio);
+
+    switch(cmd) {
+        case BIO_CTRL_GET_CLOSE:
+            return (long)BIO_get_shutdown(bio);
+        case BIO_CTRL_SET_CLOSE:
+            BIO_set_shutdown(bio, (int)num);
+            return 1;
+        case BIO_CTRL_FLUSH:
+            // we don't buffer writes, so noop
+            return 1;
+        case BIO_CTRL_EOF: {
+            switch(TYPE(target)) {
+                case T_FILE:
+                case T_OBJECT:
+                case T_STRUCT: {
+                    VALUE eof = rb_funcall(target, id_eof_p, 0);
+                    return RTEST(eof);
+                }
+                default:
+                    return 0;
+            }
+        }
+        default:
+            return 0;
+    }
+}
+
+BIO_METHOD *
+ossl_ssl_create_custom_bio_method(void)
+{
+    BIO_METHOD *m = BIO_meth_new(BIO_TYPE_MEM, "OpenSSL Ruby BIO");
+    if(m) {
+        BIO_meth_set_write(m, &ossl_ssl_custom_bio_out_write);
+        BIO_meth_set_read(m, &ossl_ssl_custom_bio_in_read);
+        BIO_meth_set_ctrl(m, &ossl_ssl_custom_bio_ctrl);
+    }
+    return m;
+}
+
+
+static BIO_METHOD *custom_bio_method = NULL;
+
+void
+ossl_ssl_set_custom_bio(SSL *ssl, VALUE target)
+{
+    if (!custom_bio_method) {
+        custom_bio_method = ossl_ssl_create_custom_bio_method();
+        id_read   = rb_intern_const("read");
+        id_write  = rb_intern_const("write");
+        id_call   = rb_intern_const("call");
+        id_eof_p  = rb_intern_const("eof?");
+    }
+
+    BIO *bio = BIO_new(custom_bio_method);
+    if(!bio)
+        rb_raise(eSSLError, "Failed to create custom BIO");
+
+    BIO_set_data(bio, (void *)target);
+#ifdef HAVE_SSL_SET0_RBIO
+    BIO_up_ref(bio);
+    SSL_set0_rbio(ssl, bio);
+    SSL_set0_wbio(ssl, bio);
+#else
+    SSL_set_bio(ssl, bio, bio);
+#endif
+}
+
+#endif

ext/openssl/ossl_ssl_custom_bio.h

@@ -0,0 +1,17 @@
+/*
+ * 'OpenSSL for Ruby' project
+ * Copyright (C) 2026  Sharon Rosner <sharon@noteflakes.com>
+ * All rights reserved.
+ */
+/*
+ * This program is licensed under the same licence as Ruby.
+ * (See the file 'COPYING'.)
+*/
+#if !defined(_OSSL_SSL_CUSTOM_BIO_H_)
+#define _OSSL_SSL_CUSTOM_BIO_H_
+
+#include "ossl.h"
+
+void ossl_ssl_set_custom_bio(SSL *ssl, VALUE target);
+
+#endif /* _OSSL_SSL_CUSTOM_BIO_H_ */