Index | Thread | Search

From:
Nick <nicklpmr@gmail.com>
Subject:
libc stdio: open_wmemstream(3) mbsnrtowcs OOB read / short-write corruption and memstream seek UB
To:
bugs@openbsd.org
Cc:
tech@openbsd.org
Date:
Tue, 17 Nov 2026 00:38:47 +0100

Download raw body.

Thread
  • Nick:

    libc stdio: open_wmemstream(3) mbsnrtowcs OOB read / short-write corruption and memstream seek UB

>Synopsis: open_wmemstream(3) swaps mbsnrtowcs(3) bounds, returns wide-char
count from _write cookie, and overflows on INT64_MIN seek
>Category: library
>Environment:
System      : OpenBSD-current (master)
Details       : lib/libc/stdio/open_wmemstream.c,v 1.12 (2026/05/08 millert)
                    lib/libc/stdio/open_memstream.c,v 1.10 (2023/07/11
claudio)
Architecture: All architectures (LP64 and ILP32)
>Description:
Several bugs remain in lib/libc/stdio/open_wmemstream.c (v1.12) and
lib/libc/stdio/open_memstream.c (v1.10):

1. Out-of-bounds read past b + l in wmemstream_write():
  mbsnrtowcs(dst, src, nmc, len, ps) takes nmc (maximum source
  bytes in *src) as its 3rd parameter and len (maximum wide
  characters written to dst) as its 4th parameter.
  wmemstream_write() computes:

      nmc = (st->size - st->pos) * sizeof(wchar_t);
      len = mbsnrtowcs(st->string + st->pos, &b, nmc, l, &st->mbs);

  passing nmc (up to >= 4096 bytes) as the source byte limit and
  l (the input byte count) as the destination wchar_t limit.
  Whenever b contains multi-byte UTF-8 characters (fewer than l
  wide characters in l bytes) and is not NUL-terminated at b[l],
  mbsnrtowcs(3) reads past b + l trying to decode l wide
  characters, either faulting on an unmapped page or decoding stale
  heap bytes from the stdio buffer.

2. Violation of stdio _write cookie return-value contract:
  __sflush() (lib/libc/stdio/fflush.c) and __sfvwrite()
  (lib/libc/stdio/fvwrite.c) pass byte count n to fp->_write and
  loop while n > 0 (n -= t, p += t), treating t <= 0 as an
  error (__SERR / EOF) and 0 < t < n as a short byte write:
  - For multi-byte UTF-8 output (e.g. fputws(L"\u00e4\u00f6\u00fc", fp),
    l = 6 bytes -> len = 3 wchar_t), returning len (3) causes
    __sflush() to advance p += 3 into the middle of a UTF-8
    continuation byte (or re-write trailing characters) and fail
    with EILSEQ / EOF.
  - For an embedded NUL wide character (fputwc(L'\0', fp)),
    mbsnrtowcs(3) stops at '\0', does not advance st->pos, and
    returns 0, causing __sflush() (if (t <= 0)) to fail with EOF
    and drop embedded NULs and any subsequent characters in the flush.
  - When a multi-byte sequence is split across two stdio buffer
    flushes, returning fewer bytes than l causes stdio to re-feed
    the already-buffered incomplete lead bytes into st->mbs.

  Replacing mbsnrtowcs(3) with a bounded mbrtowc(3) loop over
  (size_t)l bytes handles incomplete sequences ((size_t)-2 saved
  in st->mbs), embedded NUL bytes (n == 0 -> wc = L'\0', n = 1),
  and returns l bytes consumed to stdio.

3. Seek and growth integer overflows in open_wmemstream.c and
  open_memstream.c:
  - In wmemstream_seek() and memstream_seek(), base < -off triggers
    signed integer overflow UB when off == INT64_MIN (e.g. via
    fseeko(fp, INT64_MIN, SEEK_END)). On ILP32 architectures where
    size_t is 32-bit and fpos_t (off_t) is 64-bit, -INT64_MIN
    wraps to negative INT64_MIN and base < -off is evaluated in
    signed int64_t ((int64_t)base < INT64_MIN -> false), bypassing
    the negative-offset check completely.
  - Seeking before position 0 sets errno = EOVERFLOW instead of
    POSIX EINVAL, and switch (whence) lacks a default: arm
    returning EINVAL.
  - Upper bounds in *_write() and *_seek() do not guard l < 0,
    st->pos + (size_t)l wraparound, st->size * 8 overflow during
    st->size * 8 / 5 growth, or reserve 1 element for the trailing
    NUL (max_pos = (SIZE_MAX / sizeof(wchar_t)) - 1 and
    SIZE_MAX - 1).

>How-To-Repeat:
1. Multi-byte UTF-8 flush failure / corruption on OpenBSD-current:
      setlocale(LC_CTYPE, "C.UTF-8");
      FILE *fp = open_wmemstream(&buf, &sz);
      fputws(L"\u00e4\u00f6\u00fc", fp);
      fflush(fp); /* returns EOF (EILSEQ) / corrupts buf */

2. Embedded NUL failure:
      FILE *fp = open_wmemstream(&buf, &sz);
      fputwc(L'\0', fp);
      fflush(fp); /* returns EOF, sz remains 0 */

3. Out-of-bounds read past b + l:
  Place 2-byte UTF-8 \xc3\xa4 (l = 2) at the end of a page
  immediately before a PROT_NONE guard page and invoke
  wmemstream_write(cookie, tail, 2) -> immediate SIGSEGV.

4. Signed overflow in *_seek():
      fseeko(fp, INT64_MIN, SEEK_END); /* UBSan trap / ILP32 bypass */

(A standalone C reproducer repro_openbsd_wmemstream.c covering all
four cases with PROT_NONE guard pages and UBSan is accompanying this
report.)

>Fix:
Apply the following patch (against open_memstream.c,v 1.10 and
open_wmemstream.c,v 1.12):

Index: lib/libc/stdio/open_memstream.c
===================================================================
--- lib/libc/stdio/open_memstream.c
+++ lib/libc/stdio/open_memstream.c
@@ -41,12 +41,28 @@ memstream_write(void *v, const char *b,
  struct state *st = v;
  char *p;
  size_t i, end;
+ size_t max_pos = SIZE_MAX - 1;

- end = (st->pos + l);
+ if (l < 0) {
+ errno = EINVAL;
+ return (-1);
+ }
+ if (l == 0)
+ return (0);
+ if ((size_t)l > max_pos - st->pos) {
+ errno = EOVERFLOW;
+ return (-1);
+ }
+ end = st->pos + (size_t)l;

  if (end >= st->size) {
  /* 1.6 is (very) close to the golden ratio. */
- size_t sz = st->size * 8 / 5;
+ size_t sz;
+
+ if (st->size > SIZE_MAX / 8)
+ sz = end + 1;
+ else
+ sz = st->size * 8 / 5;

  if (sz < end + 1)
  sz = end + 1;
@@ -76,6 +92,7 @@ memstream_seek(void *v, fpos_t off, int
 {
  struct state *st = v;
  size_t base = 0;
+ size_t max_pos = SIZE_MAX - 1;

  switch (whence) {
  case SEEK_SET:
@@ -86,14 +103,27 @@ memstream_seek(void *v, fpos_t off, int
  case SEEK_END:
  base = st->len;
  break;
+ default:
+ errno = EINVAL;
+ return (-1);
  }

- if ((off > 0 && off > SIZE_MAX - base) || (off < 0 && base < -off)) {
- errno = EOVERFLOW;
- return (-1);
+ if (off < 0) {
+ uintmax_t neg = -(uintmax_t)off;
+
+ if (neg > base) {
+ errno = EINVAL;
+ return (-1);
+ }
+ st->pos = base - (size_t)neg;
+ } else {
+ if ((uintmax_t)off > max_pos - base) {
+ errno = EOVERFLOW;
+ return (-1);
+ }
+ st->pos = base + (size_t)off;
  }

- st->pos = base + off;
  *st->psize = MINIMUM(st->pos, st->len);

  return (st->pos);
Index: lib/libc/stdio/open_wmemstream.c
===================================================================
--- lib/libc/stdio/open_wmemstream.c
+++ lib/libc/stdio/open_wmemstream.c
@@ -41,14 +41,31 @@ static int
 wmemstream_write(void *v, const char *b, int l)
 {
  struct state *st = v;
- wchar_t *p;
- size_t nmc, len, end;
+ wchar_t *p, wc;
+ const char *src;
+ size_t rem, n, end;
+ size_t max_pos = (SIZE_MAX / sizeof(wchar_t)) - 1;

- end = (st->pos + l);
+ if (l < 0) {
+ errno = EINVAL;
+ return (-1);
+ }
+ if (l == 0)
+ return (0);
+ if ((size_t)l > max_pos - st->pos) {
+ errno = EOVERFLOW;
+ return (-1);
+ }
+ end = st->pos + (size_t)l;

  if (end >= st->size) {
  /* 1.6 is (very) close to the golden ratio. */
- size_t sz = st->size * 8 / 5;
+ size_t sz;
+
+ if (st->size > (SIZE_MAX / sizeof(wchar_t)) / 8)
+ sz = end + 1;
+ else
+ sz = st->size * 8 / 5;

  if (sz < end + 1)
  sz = end + 1;
@@ -59,11 +76,28 @@ wmemstream_write(void *v, const char *b,
  st->size = sz;
  }

- nmc = (st->size - st->pos) * sizeof(wchar_t);
- len = mbsnrtowcs(st->string + st->pos, &b, nmc, l, &st->mbs);
- if (len == (size_t)-1)
- return (-1);
- st->pos += len;
+ src = b;
+ rem = (size_t)l;
+ while (rem > 0) {
+ n = mbrtowc(&wc, src, rem, &st->mbs);
+ if (n == (size_t)-1) {
+ if (st->pos > st->len) {
+ st->len = st->pos;
+ st->string[st->len] = L'\0';
+ }
+ *st->psize = st->pos;
+ return (-1);
+ }
+ if (n == (size_t)-2)
+ break;
+ if (n == 0) {
+ wc = L'\0';
+ n = 1;
+ }
+ st->string[st->pos++] = wc;
+ src += n;
+ rem -= n;
+ }

  if (st->pos > st->len) {
  st->len = st->pos;
@@ -72,7 +106,7 @@ wmemstream_write(void *v, const char *b,

  *st->psize = st->pos;

- return (len);
+ return (l);
 }

 static fpos_t
@@ -80,6 +114,7 @@ wmemstream_seek(void *v, fpos_t off, int
 {
  struct state *st = v;
  size_t base = 0;
+ size_t max_pos = (SIZE_MAX / sizeof(wchar_t)) - 1;

  switch (whence) {
  case SEEK_SET:
@@ -90,12 +125,25 @@ wmemstream_seek(void *v, fpos_t off, int
  case SEEK_END:
  base = st->len;
  break;
+ default:
+ errno = EINVAL;
+ return (-1);
  }

- if ((off > 0 && off > (SIZE_MAX / sizeof(wchar_t)) - base) ||
-    (off < 0 && base < -off)) {
- errno = EOVERFLOW;
- return (-1);
+ if (off < 0) {
+ uintmax_t neg = -(uintmax_t)off;
+
+ if (neg > base) {
+ errno = EINVAL;
+ return (-1);
+ }
+ st->pos = base - (size_t)neg;
+ } else {
+ if ((uintmax_t)off > max_pos - base) {
+ errno = EOVERFLOW;
+ return (-1);
+ }
+ st->pos = base + (size_t)off;
  }

  /*
@@ -104,7 +152,6 @@ wmemstream_seek(void *v, fpos_t off, int
  */
  bzero(&st->mbs, sizeof(st->mbs));

- st->pos = base + off;
  *st->psize = MINIMUM(st->pos, st->len);

  return (st->pos);