Index | Thread | Search

From:
Christoph Liebender <christoph@liebender.dev>
Subject:
Re: relayd: add support for PROXY protocol in TCP relays
To:
tech@openbsd.org, reyk@openbsd.org
Date:
Thu, 18 Dec 2025 14:26:56 +0100

Download raw body.

Thread
On 11/19/25 15:54, Kirill A. Korinsky wrote:
> On Tue, 18 Nov 2025 20:55:56 +0100,
> Christoph Liebender <christoph@liebender.dev> wrote:
>>
>> [1  <text/plain; UTF-8 (7bit)>]
>> On 11/18/25 00:14, Kirill A. Korinsky wrote:
>>> On Sun, 16 Nov 2025 19:31:10 +0100,
>>> Christoph Liebender <christoph@liebender.dev> wrote:
>>>
>>> when XX was v1 and base on tcpdump relayd sends a line:
>>>
>>> PROXY TCP4 127.0.0.1 0.0.0.0 20042 8080
>>>
>>> I don't think that 0.0.0.0 is the right IP here.
>>
>> Thanks. Fixed.
>>
>>> I haven't read the structure but QUIT brings a bell. I had send a few
>>> requests and all of them contains this QUIT.
>>
>> I'm sorry for ringing your bell, but that is actually part of the
>> protocol. Some bytes of the header are QUIT in ASCII.
>>
>>>
>>> Comments in the code:
>>>
>>>> +
>>>> +int
>>>> +is_proxy_protocol_allowed(const struct protocol *p, u_int32_t flags)
>>>> +{
>>>> +	if (!p)
>>>> +		return 1;
>>>> +
>>>> +	switch (p->type) {
>>>> +	case RELAY_PROTO_HTTP:
>>>> +		if (flags & (F_PROXYV1|F_PROXYV2))
>>>> +			return 0;
>>>> +		break;
>>>> +	case RELAY_PROTO_DNS:
>>>> +		if (flags & F_PROXYV1)
>>>> +			return 0;
>>>> +		break;
>>>> +	default:
>>>> +		break;
>>>> +	}
>>>> +
>>>> +	return 1;
>>>> +}
>>>
>>> It isn't clear for me why you had difference logic for HTTP and DNS.
>>
>> I made an argument one before my previous email. But if you insist...
>> Attached is a version with the fixes and without any config
>> sanitization. It also allows v1 over UDP, and v1/v2 over HTTP. So your
>> http protocol xff will work.
>>
>>>
>>>> +#define PROXY_V2_FAM_UNSPEC 0x00
>>>> +#define PROXY_V2_FAM_TCP4   0x11
>>>> +#define PROXY_V2_FAM_UDP4   0x12
>>>> +#define	PROXY_V2_FAM_TCP6   0x21
>>>> +#define PROXY_V2_FAM_UDP6   0x22
>>>
>>> looks like poor ident
>>>
>>
>> Fixed.
>>
>> Thanks for taking a look.
>>
> 
> I had tested it against nginx and my custom implementation of proxy protocol
> in Java application, both works with v1 and v2.
> 
> So, it works and reads OK kirill@
> 
> 
>>
>> [2 relayd7.patch <text/x-patch; UTF-8 (base64)>]
>> diff --git a/usr.sbin/relayd/Makefile b/usr.sbin/relayd/Makefile
>> index 2e1ef449b04..5aacf84fc87 100644
>> --- a/usr.sbin/relayd/Makefile
>> +++ b/usr.sbin/relayd/Makefile
>> @@ -4,8 +4,9 @@ PROG=		relayd
>>   SRCS=		parse.y
>>   SRCS+=		agentx_control.c ca.c carp.c check_icmp.c check_script.c \
>>   		check_tcp.c check_tls.c config.c control.c hce.c log.c \
>> -		name2id.c pfe.c pfe_filter.c pfe_route.c proc.c relay.c \
>> -		relay_http.c relay_udp.c relayd.c shuffle.c ssl.c util.c
>> +		name2id.c pfe.c pfe_filter.c pfe_route.c proc.c \
>> +		proxy_protocol.c relay.c relay_http.c relay_udp.c relayd.c \
>> +		shuffle.c ssl.c util.c
>>   MAN=		relayd.8 relayd.conf.5
>>   
>>   LDADD=		-lagentx -levent -ltls -lssl -lcrypto -lutil
>> diff --git a/usr.sbin/relayd/parse.y b/usr.sbin/relayd/parse.y
>> index fcdfb8e92e3..8fc2016246e 100644
>> --- a/usr.sbin/relayd/parse.y
>> +++ b/usr.sbin/relayd/parse.y
>> @@ -1,6 +1,7 @@
>>   /*	$OpenBSD: parse.y,v 1.258 2024/10/28 19:56:18 tb Exp $	*/
>>   
>>   /*
>> + * Copyright (c) 2025 Christoph Liebender <christoph@liebender.dev>
>>    * Copyright (c) 2007 - 2014 Reyk Floeter <reyk@openbsd.org>
>>    * Copyright (c) 2008 Gilles Chehade <gilles@openbsd.org>
>>    * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
>> @@ -179,13 +180,13 @@ typedef struct {
>>   %token	TIMEOUT TLS TO ROUTER RTLABEL TRANSPARENT URL WITH TTL RTABLE
>>   %token	MATCH PARAMS RANDOM LEASTSTATES SRCHASH KEY CERTIFICATE PASSWORD ECDHE
>>   %token	EDH TICKETS CONNECTION CONNECTIONS CONTEXT ERRORS STATE CHANGES CHECKS
>> -%token	WEBSOCKETS PFLOG CLIENT
>> +%token	WEBSOCKETS PFLOG CLIENT PROXYPROTO V1 V2
>>   %token	<v.string>	STRING
>>   %token  <v.number>	NUMBER
>>   %type	<v.string>	context hostname interface table value path
>>   %type	<v.number>	http_type loglevel quick
>>   %type	<v.number>	dstmode flag forwardmode retry
>> -%type	<v.number>	opttls opttlsclient
>> +%type	<v.number>	opttls opttlsclient optproxyproto
>>   %type	<v.number>	redirect_proto relay_proto match pflog
>>   %type	<v.number>	action ruleaf key_option
>>   %type	<v.port>	port
>> @@ -1103,6 +1104,11 @@ proto		: relay_proto PROTO STRING	{
>>   		}
>>   		;
>>   
>> +optproxyproto	: /* empty */	{ $$ = 0; }
>> +		| PROXYPROTO V1	{ $$ = F_PROXYV1; }
>> +		| PROXYPROTO V2 { $$ = F_PROXYV2; }
>> +		;
>> +
>>   protopts_n	: /* empty */
>>   		| '{' '}'
>>   		| '{' optnl protopts_l '}'
>> @@ -1946,7 +1952,7 @@ relayoptsl	: LISTEN ON STRING port opttls {
>>   			tableport = h->port.val[0];
>>   			host_free(&al);
>>   		}
>> -		| forwardmode opttlsclient TO forwardspec dstaf {
>> +		| forwardmode opttlsclient TO forwardspec dstaf optproxyproto {
>>   			rlay->rl_conf.fwdmode = $1;
>>   			if ($1 == FWD_ROUTE) {
>>   				yyerror("no route for relays");
>> @@ -1956,6 +1962,8 @@ relayoptsl	: LISTEN ON STRING port opttls {
>>   				rlay->rl_conf.flags |= F_TLSCLIENT;
>>   				conf->sc_conf.flags |= F_TLSCLIENT;
>>   			}
>> +
>> +			rlay->rl_conf.flags |= $6;
>>   		}
>>   		| SESSION TIMEOUT NUMBER		{
>>   			if ((rlay->rl_conf.timeout.tv_sec = $3) < 0) {
>> @@ -1983,6 +1991,7 @@ relayoptsl	: LISTEN ON STRING port opttls {
>>   				free($2);
>>   				YYERROR;
>>   			}
>> +
>>   			p->flags |= F_USED;
>>   			rlay->rl_conf.proto = p->id;
>>   			rlay->rl_proto = p;
>> @@ -2479,6 +2488,7 @@ lookup(char *s)
>>   		{ "prefork",		PREFORK },
>>   		{ "priority",		PRIORITY },
>>   		{ "protocol",		PROTO },
>> +		{ "proxy-protocol",	PROXYPROTO },
>>   		{ "query",		QUERYSTR },
>>   		{ "quick",		QUICK },
>>   		{ "random",		RANDOM },
>> @@ -2518,6 +2528,8 @@ lookup(char *s)
>>   		{ "transparent",	TRANSPARENT },
>>   		{ "ttl",		TTL },
>>   		{ "url",		URL },
>> +		{ "v1",			V1 },
>> +		{ "v2",			V2 },
>>   		{ "value",		VALUE },
>>   		{ "websockets",		WEBSOCKETS },
>>   		{ "with",		WITH }
>> diff --git a/usr.sbin/relayd/proxy_protocol.c b/usr.sbin/relayd/proxy_protocol.c
>> new file mode 100644
>> index 00000000000..d832d3f6678
>> --- /dev/null
>> +++ b/usr.sbin/relayd/proxy_protocol.c
>> @@ -0,0 +1,142 @@
>> +/*
>> + * Copyright (c) 2025 Christoph Liebender <christoph@liebender.dev>
>> + *
>> + * Permission to use, copy, modify, and distribute this software for any
>> + * purpose with or without fee is hereby granted, provided that the above
>> + * copyright notice and this permission notice appear in all copies.
>> + *
>> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
>> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
>> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
>> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
>> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
>> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
>> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
>> + */
>> +
>> +#include "relayd.h"
>> +
>> +#define PROXY_V2_CMD_PROXY 0x01
>> +
>> +#define PROXY_V2_FAM_UNSPEC 0x00
>> +#define PROXY_V2_FAM_TCP4   0x11
>> +#define PROXY_V2_FAM_UDP4   0x12
>> +#define PROXY_V2_FAM_TCP6   0x21
>> +#define PROXY_V2_FAM_UDP6   0x22
>> +
>> +static const u_int8_t PROXY_V2_SIG[12] = {
>> +	0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A
>> +};
>> +
>> +struct proxy_v2_hdr {
>> +	u_int8_t sig[12];
>> +	u_int8_t ver_cmd;
>> +	u_int8_t fam;
>> +	u_int16_t len;
>> +};
>> +
>> +union proxy_v2_addr {
>> +	struct {
>> +		u_int32_t src_addr;
>> +		u_int32_t dst_addr;
>> +		in_port_t src_port;
>> +		in_port_t dst_port;
>> +	} ipv4_addr;
>> +	struct {
>> +		u_int8_t  src_addr[16];
>> +		u_int8_t  dst_addr[16];
>> +		in_port_t src_port;
>> +		in_port_t dst_port;
>> +	} ipv6_addr;
>> +};
>> +
>> +int
>> +proxy_protocol_v1(const struct rsession *con, struct evbuffer *dstout)
>> +{
>> +	char ibuf[128], obuf[128];
>> +	const char *proxyproto;
>> +	int ret;
>> +
>> +	bzero(&ibuf, sizeof(ibuf));
>> +	bzero(&obuf, sizeof(obuf));
>> +	print_host(&con->se_in.ss, ibuf, sizeof(ibuf));
>> +	print_host(&con->se_sockname, obuf, sizeof(obuf));
>> +
>> +	if (con->se_relay->rl_conf.flags & F_UDP)
>> +		proxyproto = "UNKNOWN";
>> +	else {
>> +		switch (con->se_in.ss.ss_family) {
>> +		case AF_INET:
>> +			proxyproto = "TCP4";
>> +			break;
>> +		case AF_INET6:
>> +			proxyproto = "TCP6";
>> +			break;
>> +		default:
>> +			proxyproto = "UNKNOWN";
>> +			break;
>> +		}
>> +	}
>> +
>> +	ret = evbuffer_add_printf(dstout,
>> +	    "PROXY %s %s %s %d %d\r\n", proxyproto, ibuf, obuf,
>> +	    ntohs(con->se_in.port), ntohs(con->se_relay->rl_conf.port));
>> +
>> +	return ret == -1 ? -1 : 0;
>> +}
>> +
>> +int
>> +proxy_protocol_v2(const struct rsession *con, struct evbuffer *dstout)
>> +{
>> +	union proxy_v2_addr 	       addr;
>> +	struct proxy_v2_hdr 	       hdr;
>> +	const struct relay_config     *conf = &con->se_relay->rl_conf;
>> +	const struct sockaddr_storage *srcss = &con->se_in.ss;
>> +	const struct sockaddr_storage *dstss = &con->se_sockname;
>> +	int 			       error;
>> +	in_port_t		       srcport = con->se_in.port;
>> +	in_port_t		       dstport = conf->port;
>> +	u_int16_t 		       len;
>> +
>> +	bcopy(PROXY_V2_SIG, hdr.sig, sizeof(hdr.sig));
>> +	hdr.ver_cmd = 0x20 | PROXY_V2_CMD_PROXY;
>> +
>> +	switch (dstss->ss_family) {
>> +	case AF_INET:
>> +		hdr.fam = (conf->flags & F_UDP) ?
>> +			PROXY_V2_FAM_UDP4 : PROXY_V2_FAM_TCP4;
>> +		len = sizeof(addr.ipv4_addr);
>> +		addr.ipv4_addr.src_addr =
>> +		    ((const struct sockaddr_in *)srcss)->sin_addr.s_addr;
>> +		addr.ipv4_addr.dst_addr =
>> +		    ((const struct sockaddr_in *)dstss)->sin_addr.s_addr;
>> +		addr.ipv4_addr.src_port = srcport;
>> +		addr.ipv4_addr.dst_port = dstport;
>> +		break;
>> +	case AF_INET6:
>> +		hdr.fam = (conf->flags & F_UDP) ?
>> +			PROXY_V2_FAM_UDP6 : PROXY_V2_FAM_TCP6;
>> +		len = sizeof(addr.ipv6_addr);
>> +		bcopy(&((const struct sockaddr_in6 *)srcss)->sin6_addr,
>> +		    addr.ipv6_addr.src_addr, sizeof(addr.ipv6_addr.src_addr));
>> +		bcopy(&((const struct sockaddr_in6 *)dstss)->sin6_addr,
>> +		    addr.ipv6_addr.dst_addr, sizeof(addr.ipv6_addr.dst_addr));
>> +		addr.ipv6_addr.src_port = srcport;
>> +		addr.ipv6_addr.dst_port = dstport;
>> +		break;
>> +	default:
>> +		hdr.fam = 0x00;
>> +		len = 0;
>> +		break;
>> +	}
>> +
>> +	hdr.len = htons(len);
>> +
>> +	if ((error = evbuffer_add(dstout, &hdr, sizeof(hdr))) != 0)
>> +		return error;
>> +
>> +	if (len > 0 && (error = evbuffer_add(dstout, &addr, len)) != 0)
>> +		return error;
>> +
>> +	return 0;
>> +}
>> diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
>> index 6d0970802c5..d705ac1552a 100644
>> --- a/usr.sbin/relayd/relay.c
>> +++ b/usr.sbin/relayd/relay.c
>> @@ -1,6 +1,7 @@
>>   /*	$OpenBSD: relay.c,v 1.260 2024/10/28 19:56:18 tb Exp $	*/
>>   
>>   /*
>> + * Copyright (c) 2025 Christoph Liebender <christoph@liebender.dev>
>>    * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org>
>>    *
>>    * Permission to use, copy, modify, and distribute this software for any
>> @@ -767,6 +768,19 @@ relay_connected(int fd, short sig, void *arg)
>>   		    "failed to allocate output buffer event", 0);
>>   		return;
>>   	}
>> +
>> +	error = 0;
>> +
>> +	if (rlay->rl_conf.flags & F_PROXYV1)
>> +		error = proxy_protocol_v1(con, bev->output);
>> +	else if (rlay->rl_conf.flags & F_PROXYV2)
>> +		error = proxy_protocol_v2(con, bev->output);
>> +
>> +	if (error) {
>> +		relay_abort_http(con, 500, "failed to write PROXY header", 0);
>> +		return;
>> +	}
>> +
>>   	/* write pending output buffer now */
>>   	if (bufferevent_write_buffer(bev, con->se_out.output)) {
>>   		relay_abort_http(con, 500, strerror(errno), 0);
>> diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
>> index 29a13efbe8a..6f3a35c0fca 100644
>> --- a/usr.sbin/relayd/relayd.conf.5
>> +++ b/usr.sbin/relayd/relayd.conf.5
>> @@ -1,5 +1,6 @@
>>   .\"	$OpenBSD: relayd.conf.5,v 1.213 2025/07/08 14:26:45 schwarze Exp $
>>   .\"
>> +.\" Copyright (c) 2025 Christoph Liebender <christoph@liebender.dev>
>>   .\" Copyright (c) 2006 - 2016 Reyk Floeter <reyk@openbsd.org>
>>   .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
>>   .\"
>> @@ -673,6 +674,13 @@ option will be used as a tolerance for failed
>>   host connections; the connection will be retried for
>>   .Ar number
>>   more times.
>> +.It Ic proxy-protocol Pq Ic v1 Ns | Ns Ic v2
>> +Upon connection to the destination,
>> +.Xr relayd 8
>> +will prepend a PROXY protocol header of the specified version to the relayed
>> +data, with the IP address and port of the remote host as well as the
>> +.Xr relayd 8
>> +server.
>>   .El
>>   .It Xo
>>   .Ic forward to
>> diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
>> index 3b5c3987f93..4fca5b5329f 100644
>> --- a/usr.sbin/relayd/relayd.h
>> +++ b/usr.sbin/relayd/relayd.h
>> @@ -1,6 +1,7 @@
>>   /*	$OpenBSD: relayd.h,v 1.276 2024/10/28 19:56:18 tb Exp $	*/
>>   
>>   /*
>> + * Copyright (c) 2025 Christoph Liebender <christoph@liebender.dev>
>>    * Copyright (c) 2006 - 2016 Reyk Floeter <reyk@openbsd.org>
>>    * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
>>    * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
>> @@ -402,6 +403,8 @@ union hashkey {
>>   #define F_HASHKEY		0x08000000
>>   #define F_AGENTX_TRAPONLY	0x10000000
>>   #define F_PFLOG			0x20000000
>> +#define F_PROXYV1		0x40000000
>> +#define F_PROXYV2		0x80000000
>>   
>>   #define F_BITS								\
>>   	"\10\01DISABLE\02BACKUP\03USED\04DOWN\05ADD\06DEL\07CHANGED"	\
>> @@ -1166,7 +1169,7 @@ const char *table_check(enum table_check);
>>   const char *relay_state(enum relay_state);
>>   #endif
>>   const char *print_availability(u_long, u_long);
>> -const char *print_host(struct sockaddr_storage *, char *, size_t);
>> +const char *print_host(const struct sockaddr_storage *, char *, size_t);
>>   const char *print_time(struct timeval *, struct timeval *, char *, size_t);
>>   const char *printb_flags(const u_int32_t, const char *);
>>   void	 getmonotime(struct timeval *);
>> @@ -1476,4 +1479,8 @@ int	 config_getrelay(struct relayd *, struct imsg *);
>>   int	 config_getrelaytable(struct relayd *, struct imsg *);
>>   int	 config_getrelayfd(struct relayd *, struct imsg *);
>>   
>> +/* proxy_protocol.c */
>> +int	proxy_protocol_v1(const struct rsession *, struct evbuffer *);
>> +int	proxy_protocol_v2(const struct rsession *, struct evbuffer *);
>> +
>>   #endif /* RELAYD_H */
>> diff --git a/usr.sbin/relayd/util.c b/usr.sbin/relayd/util.c
>> index 751e82c43c8..c82d254821f 100644
>> --- a/usr.sbin/relayd/util.c
>> +++ b/usr.sbin/relayd/util.c
>> @@ -214,9 +214,9 @@ print_availability(u_long cnt, u_long up)
>>   }
>>   
>>   const char *
>> -print_host(struct sockaddr_storage *ss, char *buf, size_t len)
>> +print_host(const struct sockaddr_storage *ss, char *buf, size_t len)
>>   {
>> -	if (getnameinfo((struct sockaddr *)ss, ss->ss_len,
>> +	if (getnameinfo((const struct sockaddr *)ss, ss->ss_len,
>>   	    buf, len, NULL, 0, NI_NUMERICHOST) != 0) {
>>   		buf[0] = '\0';
>>   		return (NULL);
> 

Ping!