Index | Thread | Search

From:
Job Snijders <job@bsd.nl>
Subject:
rpki-client: compressed canonical cache representations
To:
tech@openbsd.org
Date:
Tue, 30 Dec 2025 12:35:45 +0000

Download raw body.

Thread
  • Job Snijders:

    rpki-client: compressed canonical cache representations

Turns out CCR data is highly compressable (~ 50%). At present CCRs are ~
10 MB in compressed form. Rename the CCR output file to align with other
outputs and, while there, remove the cautionary note. I think we're now.

OK?

Index: ccr.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/ccr.c,v
diff -u -p -r1.32 ccr.c
--- ccr.c	30 Dec 2025 09:04:09 -0000	1.32
+++ ccr.c	30 Dec 2025 12:26:15 -0000
@@ -24,6 +24,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <zlib.h>
 
 #include <openssl/asn1.h>
 #include <openssl/asn1t.h>
@@ -848,12 +849,25 @@ ccr_vrp_cmp(const struct vrp *a, const s
 RB_GENERATE(ccr_vrp_tree, vrp, entry, ccr_vrp_cmp);
 
 int
-output_ccr_der(FILE *out, struct validation_data *vd, struct stats *st)
+output_ccr(FILE *out, struct validation_data *vd, struct stats *st)
 {
-	if (fwrite(vd->ccr.der, vd->ccr.der_len, 1, out) != 1)
-		err(1, "fwrite");
+	int fd, rc = -1;
+	gzFile gzf;
 
-	return 0;
+	if ((fd = dup(fileno(out))) == -1)
+		return -1;
+
+	if ((gzf = gzdopen(fd, "w")) == NULL)
+		goto out;
+
+	if (gzfwrite(vd->ccr.der, vd->ccr.der_len, 1, gzf) != 1)
+		goto out;
+
+	rc = 0;
+ out:
+	if (gzclose(gzf) != Z_OK)
+		return -1;
+	return rc;
 }
 
 static void
Index: encoding.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/encoding.c,v
diff -u -p -r1.14 encoding.c
--- encoding.c	9 Sep 2025 08:23:24 -0000	1.14
+++ encoding.c	30 Dec 2025 12:26:15 -0000
@@ -24,6 +24,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <zlib.h>
 
 #include <openssl/evp.h>
 
@@ -71,6 +72,58 @@ err:
 	free(buf);
 	errno = saved_errno;
 	return NULL;
+}
+
+#define GZIP_CHUNK_SIZE	(32 * 1024)
+
+/*
+ * One-shot gzip data decompressor.
+ * Return 0 on success, -1 on error.
+ */
+int
+inflate_buffer(uint8_t **inbuf, size_t *inlen)
+{
+	z_stream zs;
+	uint8_t *buf = NULL, *nbuf;
+	size_t buf_len;
+	int rc = -1, zret;
+
+	memset(&zs, 0, sizeof(zs));
+
+	zs.avail_in = *inlen;
+	zs.next_in = *inbuf;
+
+	if (inflateInit2(&zs, MAX_WBITS + 16) != Z_OK)
+		goto out;
+
+	buf_len = *inlen * 2;
+	do {
+		buf_len += GZIP_CHUNK_SIZE;
+		if ((nbuf = realloc(buf, buf_len)) == NULL)
+			err(1, NULL);
+		buf = nbuf;
+		zs.next_out = buf + zs.total_out;
+		zs.avail_out = buf_len - zs.total_out;
+
+		zret = inflate(&zs, Z_NO_FLUSH);
+		if (zret != Z_OK && zret != Z_STREAM_END)
+			goto out;
+	} while (zs.avail_out == 0);
+
+	if ((nbuf = realloc(buf, zs.total_out)) == NULL)
+		err(1, NULL);
+	buf = nbuf;
+
+	*inbuf = buf;
+	buf = NULL;
+	*inlen = zs.total_out;
+
+	rc = 0;
+ out:
+	free(buf);
+	if (inflateEnd(&zs) != Z_OK)
+		return -1;
+	return rc;
 }
 
 /*
Index: extern.h
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v
diff -u -p -r1.268 extern.h
--- extern.h	18 Nov 2025 14:04:45 -0000	1.268
+++ extern.h	30 Dec 2025 12:26:15 -0000
@@ -203,6 +203,7 @@ enum rtype {
 	RTYPE_GEOFEED,
 	RTYPE_SPL,
 	RTYPE_CCR,
+	RTYPE_CCR_GZ,
 };
 
 enum location {
@@ -937,9 +938,14 @@ void		 rrdp_fetch(unsigned int, const ch
 void		 rrdp_abort(unsigned int);
 void		 rrdp_http_done(unsigned int, enum http_result, const char *);
 
-/* Encoding functions for hex and base64. */
+
+/* File loading and decompression functions. */
 
 unsigned char	*load_file(const char *, size_t *);
+int		 inflate_buffer(uint8_t **, size_t *);
+
+/* Encoding functions for hex and base64. */
+
 int		 base64_decode_len(size_t, size_t *);
 int		 base64_decode(const unsigned char *, size_t,
 		    unsigned char **, size_t *);
@@ -1018,12 +1024,12 @@ extern int	 outformats;
 int		 outputfiles(struct validation_data *, struct stats *);
 int		 outputheader(FILE *, struct validation_data *, struct stats *);
 int		 output_bgpd(FILE *, struct validation_data *, struct stats *);
+int		 output_ccr(FILE *, struct validation_data *, struct stats *);
 int		 output_bird(FILE *, struct validation_data *, struct stats *);
 int		 output_csv(FILE *, struct validation_data *, struct stats *);
 int		 output_json(FILE *, struct validation_data *, struct stats *);
 int		 output_ometric(FILE *, struct validation_data *,
 		    struct stats *);
-int		 output_ccr_der(FILE *, struct validation_data *, struct stats *);
 
 /*
  * Canonical Cache Representation
Index: filemode.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/filemode.c,v
diff -u -p -r1.73 filemode.c
--- filemode.c	5 Dec 2025 08:41:32 -0000	1.73
+++ filemode.c	30 Dec 2025 12:26:16 -0000
@@ -457,6 +457,17 @@ proc_parser_file(char *file, unsigned ch
 		}
 	}
 
+	type = rtype_from_file_extension(file);
+	if (type == RTYPE_CCR_GZ) {
+		if (inflate_buffer(&buf, &len) == -1) {
+			warnx("%s: gzip decompression failed", file);
+			goto out;
+		}
+		type = RTYPE_CCR;
+	}
+	if (type == RTYPE_INVALID)
+		type = rtype_from_der(file, buf, len);
+
 	if (!EVP_Digest(buf, len, filehash, NULL, EVP_sha256(), NULL))
 		errx(1, "EVP_Digest failed in %s", __func__);
 
@@ -472,10 +483,6 @@ proc_parser_file(char *file, unsigned ch
 	}
 
 	free(hash);
-
-	type = rtype_from_file_extension(file);
-	if (type == RTYPE_INVALID)
-		type = rtype_from_der(file, buf, len);
 
 	switch (type) {
 	case RTYPE_ASPA:
Index: mft.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v
diff -u -p -r1.134 mft.c
--- mft.c	2 Dec 2025 06:54:08 -0000	1.134
+++ mft.c	30 Dec 2025 12:26:16 -0000
@@ -91,8 +91,12 @@ rtype_from_file_extension(const char *fn
 		return RTYPE_GEOFEED;
 	if (strcasecmp(fn + sz - 4, ".spl") == 0)
 		return RTYPE_SPL;
+	if (strcasecmp(fn + sz - 3, "ccr") == 0)
+		return RTYPE_CCR;
 	if (strcasecmp(fn + sz - 4, ".ccr") == 0)
 		return RTYPE_CCR;
+	if (strcasecmp(fn + sz - 6, "ccr.gz") == 0)
+		return RTYPE_CCR_GZ;
 
 	return RTYPE_INVALID;
 }
Index: output.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/output.c,v
diff -u -p -r1.45 output.c
--- output.c	13 Nov 2025 15:18:53 -0000	1.45
+++ output.c	30 Dec 2025 12:26:16 -0000
@@ -70,7 +70,7 @@ static const struct outputs {
 	{ FORMAT_CSV, "csv", output_csv },
 	{ FORMAT_JSON, "json", output_json },
 	{ FORMAT_OMETRIC, "metrics", output_ometric },
-	{ FORMAT_CCR, "rpki.ccr", output_ccr_der },
+	{ FORMAT_CCR, "ccr.gz", output_ccr },
 	{ 0, NULL, NULL }
 };
 
Index: rpki-client.8
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/rpki-client.8,v
diff -u -p -r1.134 rpki-client.8
--- rpki-client.8	2 Dec 2025 12:21:39 -0000	1.134
+++ rpki-client.8	30 Dec 2025 12:26:16 -0000
@@ -116,11 +116,12 @@ and
 .Fl -address
 flags and connect with rsync-protocol locations.
 .It Fl f Ar
-Attempt to decode and validate the RPKI object in
+Attempt to decode and validate the signed RPKI object or CCR in
 .Ar file
 using the cache stored in
 .Ar cachedir
 and print human-readable information about the object.
+Gzip compressed canonical cache representations are inflated on the fly.
 If
 .Ar file
 is an rsync:// URI, the corresponding file from the cache will be used.
@@ -245,9 +246,8 @@ Defaults to
 .Pp
 By default
 .Nm
-outputs validated payloads in
-.Fl o
-(OpenBGPD compatible) format.
+outputs validated payloads in OpenBGPD format and in compressed canonical
+cache representation format.
 .Pp
 .Nm
 should be run hourly by
@@ -302,7 +302,7 @@ utilizes the following environment varia
 URL of HTTP proxy to use.
 .El
 .Sh FILES
-.Bl -tag -width "/var/db/rpki-client/openbgpd" -compact
+.Bl -tag -width Ds -compact
 .It Pa /etc/rpki/*.tal
 default TAL files used unless
 .Fl t Ar tal
@@ -320,9 +320,8 @@ is specified.
 cached repository data.
 .It Pa /var/db/rpki-client/openbgpd
 default roa-set output file.
-.It Pa /var/db/rpki-client/rpki.ccr
-DER-encoded canonical cache representation file.
-This facility is experimental and is still subject to change.
+.It Pa /var/db/rpki-client/ccr.gz
+Gzip compressed canonical cache representation.
 .El
 .Sh EXIT STATUS
 .Ex -std