From: Job Snijders Subject: rpki-client: compressed canonical cache representations To: tech@openbsd.org Date: Tue, 30 Dec 2025 12:35:45 +0000 Turns out CCR data is highly compressable (~ 50%). At present CCRs are ~ 10 MB in compressed form. Rename the CCR output file to align with other outputs and, while there, remove the cautionary note. I think we're now. OK? Index: ccr.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/ccr.c,v diff -u -p -r1.32 ccr.c --- ccr.c 30 Dec 2025 09:04:09 -0000 1.32 +++ ccr.c 30 Dec 2025 12:26:15 -0000 @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -848,12 +849,25 @@ ccr_vrp_cmp(const struct vrp *a, const s RB_GENERATE(ccr_vrp_tree, vrp, entry, ccr_vrp_cmp); int -output_ccr_der(FILE *out, struct validation_data *vd, struct stats *st) +output_ccr(FILE *out, struct validation_data *vd, struct stats *st) { - if (fwrite(vd->ccr.der, vd->ccr.der_len, 1, out) != 1) - err(1, "fwrite"); + int fd, rc = -1; + gzFile gzf; - return 0; + if ((fd = dup(fileno(out))) == -1) + return -1; + + if ((gzf = gzdopen(fd, "w")) == NULL) + goto out; + + if (gzfwrite(vd->ccr.der, vd->ccr.der_len, 1, gzf) != 1) + goto out; + + rc = 0; + out: + if (gzclose(gzf) != Z_OK) + return -1; + return rc; } static void Index: encoding.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/encoding.c,v diff -u -p -r1.14 encoding.c --- encoding.c 9 Sep 2025 08:23:24 -0000 1.14 +++ encoding.c 30 Dec 2025 12:26:15 -0000 @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -71,6 +72,58 @@ err: free(buf); errno = saved_errno; return NULL; +} + +#define GZIP_CHUNK_SIZE (32 * 1024) + +/* + * One-shot gzip data decompressor. + * Return 0 on success, -1 on error. + */ +int +inflate_buffer(uint8_t **inbuf, size_t *inlen) +{ + z_stream zs; + uint8_t *buf = NULL, *nbuf; + size_t buf_len; + int rc = -1, zret; + + memset(&zs, 0, sizeof(zs)); + + zs.avail_in = *inlen; + zs.next_in = *inbuf; + + if (inflateInit2(&zs, MAX_WBITS + 16) != Z_OK) + goto out; + + buf_len = *inlen * 2; + do { + buf_len += GZIP_CHUNK_SIZE; + if ((nbuf = realloc(buf, buf_len)) == NULL) + err(1, NULL); + buf = nbuf; + zs.next_out = buf + zs.total_out; + zs.avail_out = buf_len - zs.total_out; + + zret = inflate(&zs, Z_NO_FLUSH); + if (zret != Z_OK && zret != Z_STREAM_END) + goto out; + } while (zs.avail_out == 0); + + if ((nbuf = realloc(buf, zs.total_out)) == NULL) + err(1, NULL); + buf = nbuf; + + *inbuf = buf; + buf = NULL; + *inlen = zs.total_out; + + rc = 0; + out: + free(buf); + if (inflateEnd(&zs) != Z_OK) + return -1; + return rc; } /* Index: extern.h =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v diff -u -p -r1.268 extern.h --- extern.h 18 Nov 2025 14:04:45 -0000 1.268 +++ extern.h 30 Dec 2025 12:26:15 -0000 @@ -203,6 +203,7 @@ enum rtype { RTYPE_GEOFEED, RTYPE_SPL, RTYPE_CCR, + RTYPE_CCR_GZ, }; enum location { @@ -937,9 +938,14 @@ void rrdp_fetch(unsigned int, const ch void rrdp_abort(unsigned int); void rrdp_http_done(unsigned int, enum http_result, const char *); -/* Encoding functions for hex and base64. */ + +/* File loading and decompression functions. */ unsigned char *load_file(const char *, size_t *); +int inflate_buffer(uint8_t **, size_t *); + +/* Encoding functions for hex and base64. */ + int base64_decode_len(size_t, size_t *); int base64_decode(const unsigned char *, size_t, unsigned char **, size_t *); @@ -1018,12 +1024,12 @@ extern int outformats; int outputfiles(struct validation_data *, struct stats *); int outputheader(FILE *, struct validation_data *, struct stats *); int output_bgpd(FILE *, struct validation_data *, struct stats *); +int output_ccr(FILE *, struct validation_data *, struct stats *); int output_bird(FILE *, struct validation_data *, struct stats *); int output_csv(FILE *, struct validation_data *, struct stats *); int output_json(FILE *, struct validation_data *, struct stats *); int output_ometric(FILE *, struct validation_data *, struct stats *); -int output_ccr_der(FILE *, struct validation_data *, struct stats *); /* * Canonical Cache Representation Index: filemode.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/filemode.c,v diff -u -p -r1.73 filemode.c --- filemode.c 5 Dec 2025 08:41:32 -0000 1.73 +++ filemode.c 30 Dec 2025 12:26:16 -0000 @@ -457,6 +457,17 @@ proc_parser_file(char *file, unsigned ch } } + type = rtype_from_file_extension(file); + if (type == RTYPE_CCR_GZ) { + if (inflate_buffer(&buf, &len) == -1) { + warnx("%s: gzip decompression failed", file); + goto out; + } + type = RTYPE_CCR; + } + if (type == RTYPE_INVALID) + type = rtype_from_der(file, buf, len); + if (!EVP_Digest(buf, len, filehash, NULL, EVP_sha256(), NULL)) errx(1, "EVP_Digest failed in %s", __func__); @@ -472,10 +483,6 @@ proc_parser_file(char *file, unsigned ch } free(hash); - - type = rtype_from_file_extension(file); - if (type == RTYPE_INVALID) - type = rtype_from_der(file, buf, len); switch (type) { case RTYPE_ASPA: Index: mft.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v diff -u -p -r1.134 mft.c --- mft.c 2 Dec 2025 06:54:08 -0000 1.134 +++ mft.c 30 Dec 2025 12:26:16 -0000 @@ -91,8 +91,12 @@ rtype_from_file_extension(const char *fn return RTYPE_GEOFEED; if (strcasecmp(fn + sz - 4, ".spl") == 0) return RTYPE_SPL; + if (strcasecmp(fn + sz - 3, "ccr") == 0) + return RTYPE_CCR; if (strcasecmp(fn + sz - 4, ".ccr") == 0) return RTYPE_CCR; + if (strcasecmp(fn + sz - 6, "ccr.gz") == 0) + return RTYPE_CCR_GZ; return RTYPE_INVALID; } Index: output.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output.c,v diff -u -p -r1.45 output.c --- output.c 13 Nov 2025 15:18:53 -0000 1.45 +++ output.c 30 Dec 2025 12:26:16 -0000 @@ -70,7 +70,7 @@ static const struct outputs { { FORMAT_CSV, "csv", output_csv }, { FORMAT_JSON, "json", output_json }, { FORMAT_OMETRIC, "metrics", output_ometric }, - { FORMAT_CCR, "rpki.ccr", output_ccr_der }, + { FORMAT_CCR, "ccr.gz", output_ccr }, { 0, NULL, NULL } }; Index: rpki-client.8 =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rpki-client.8,v diff -u -p -r1.134 rpki-client.8 --- rpki-client.8 2 Dec 2025 12:21:39 -0000 1.134 +++ rpki-client.8 30 Dec 2025 12:26:16 -0000 @@ -116,11 +116,12 @@ and .Fl -address flags and connect with rsync-protocol locations. .It Fl f Ar -Attempt to decode and validate the RPKI object in +Attempt to decode and validate the signed RPKI object or CCR in .Ar file using the cache stored in .Ar cachedir and print human-readable information about the object. +Gzip compressed canonical cache representations are inflated on the fly. If .Ar file is an rsync:// URI, the corresponding file from the cache will be used. @@ -245,9 +246,8 @@ Defaults to .Pp By default .Nm -outputs validated payloads in -.Fl o -(OpenBGPD compatible) format. +outputs validated payloads in OpenBGPD format and in compressed canonical +cache representation format. .Pp .Nm should be run hourly by @@ -302,7 +302,7 @@ utilizes the following environment varia URL of HTTP proxy to use. .El .Sh FILES -.Bl -tag -width "/var/db/rpki-client/openbgpd" -compact +.Bl -tag -width Ds -compact .It Pa /etc/rpki/*.tal default TAL files used unless .Fl t Ar tal @@ -320,9 +320,8 @@ is specified. cached repository data. .It Pa /var/db/rpki-client/openbgpd default roa-set output file. -.It Pa /var/db/rpki-client/rpki.ccr -DER-encoded canonical cache representation file. -This facility is experimental and is still subject to change. +.It Pa /var/db/rpki-client/ccr.gz +Gzip compressed canonical cache representation. .El .Sh EXIT STATUS .Ex -std