From: Theo Buehler <tb@theobuehler.org>
Subject: libtls: missing length checks before BIO_new_mem_buf()
To: Michael Forney <mforney@mforney.org>
Cc: tech@openbsd.org, jsing@openbsd.org, bcook@openbsd.org
Date: Thu, 16 Apr 2026 05:30:56 +0200

> I think there are three places (one in tls_keypair_load_cert() and
> two in tls_signer_add_keypair_mem()) where the size passed to
> BIO_new_mem_buf() could in principle be larger than INT_MAX. This can
> overflow to be negative or be truncated to remain positive. In the first
> case we'd have no relevant errno set.
> 
> We should fix that at least by adding errors for length > INT_MAX
> before BIO_new_mem_buf(), and perhaps we should also consider
> not loading files of such preposterous size in the first place.

The minimal diff is this:

Index: tls_keypair.c
===================================================================
RCS file: /cvs/src/lib/libtls/tls_keypair.c,v
diff -u -p -r1.9 tls_keypair.c
--- tls_keypair.c	26 Mar 2024 06:24:52 -0000	1.9
+++ tls_keypair.c	16 Apr 2026 03:17:48 -0000
@@ -148,6 +148,11 @@ tls_keypair_load_cert(struct tls_keypair
 		    "keypair has no certificate");
 		goto err;
 	}
+	if (keypair->cert_len > INT_MAX) {
+		tls_error_setx(error, TLS_ERROR_INVALID_ARGUMENT,
+		    "certificate too long");
+		goto err;
+	}
 	if ((cert_bio = BIO_new_mem_buf(keypair->cert_mem,
 	    keypair->cert_len)) == NULL) {
 		tls_error_set(error, TLS_ERROR_UNKNOWN,
Index: tls_signer.c
===================================================================
RCS file: /cvs/src/lib/libtls/tls_signer.c,v
diff -u -p -r1.13 tls_signer.c
--- tls_signer.c	11 Jun 2024 16:35:24 -0000	1.13
+++ tls_signer.c	16 Apr 2026 03:17:48 -0000
@@ -99,6 +99,11 @@ tls_signer_add_keypair_mem(struct tls_si
 	char *hash = NULL;
 
 	/* Compute certificate hash */
+	if (cert_len > INT_MAX) {
+		tls_error_setx(&signer->error, TLS_ERROR_INVALID_ARGUMENT,
+		    "certificate too long");
+		goto err;
+	}
 	if ((bio = BIO_new_mem_buf(cert, cert_len)) == NULL) {
 		tls_error_setx(&signer->error, TLS_ERROR_UNKNOWN,
 		    "failed to create certificate bio");
@@ -124,6 +129,11 @@ tls_signer_add_keypair_mem(struct tls_si
 	bio = NULL;
 
 	/* Read private key */
+	if (key_len > INT_MAX) {
+		tls_error_setx(&signer->error, TLS_ERROR_INVALID_ARGUMENT,
+		    "private key too long");
+		goto err;
+	}
 	if ((bio = BIO_new_mem_buf(key, key_len)) == NULL) {
 		tls_error_setx(&signer->error, TLS_ERROR_UNKNOWN,
 		    "failed to create key bio");