Download raw body.
rpki-client: compressed canonical cache representations
Turns out CCR data is highly compressable (~ 50%). At present CCRs are ~
10 MB in compressed form. Rename the CCR output file to align with other
outputs and, while there, remove the cautionary note. I think we're now.
OK?
Index: ccr.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/ccr.c,v
diff -u -p -r1.32 ccr.c
--- ccr.c 30 Dec 2025 09:04:09 -0000 1.32
+++ ccr.c 30 Dec 2025 12:26:15 -0000
@@ -24,6 +24,7 @@
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
+#include <zlib.h>
#include <openssl/asn1.h>
#include <openssl/asn1t.h>
@@ -848,12 +849,25 @@ ccr_vrp_cmp(const struct vrp *a, const s
RB_GENERATE(ccr_vrp_tree, vrp, entry, ccr_vrp_cmp);
int
-output_ccr_der(FILE *out, struct validation_data *vd, struct stats *st)
+output_ccr(FILE *out, struct validation_data *vd, struct stats *st)
{
- if (fwrite(vd->ccr.der, vd->ccr.der_len, 1, out) != 1)
- err(1, "fwrite");
+ int fd, rc = -1;
+ gzFile gzf;
- return 0;
+ if ((fd = dup(fileno(out))) == -1)
+ return -1;
+
+ if ((gzf = gzdopen(fd, "w")) == NULL)
+ goto out;
+
+ if (gzfwrite(vd->ccr.der, vd->ccr.der_len, 1, gzf) != 1)
+ goto out;
+
+ rc = 0;
+ out:
+ if (gzclose(gzf) != Z_OK)
+ return -1;
+ return rc;
}
static void
Index: encoding.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/encoding.c,v
diff -u -p -r1.14 encoding.c
--- encoding.c 9 Sep 2025 08:23:24 -0000 1.14
+++ encoding.c 30 Dec 2025 12:26:15 -0000
@@ -24,6 +24,7 @@
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
+#include <zlib.h>
#include <openssl/evp.h>
@@ -71,6 +72,58 @@ err:
free(buf);
errno = saved_errno;
return NULL;
+}
+
+#define GZIP_CHUNK_SIZE (32 * 1024)
+
+/*
+ * One-shot gzip data decompressor.
+ * Return 0 on success, -1 on error.
+ */
+int
+inflate_buffer(uint8_t **inbuf, size_t *inlen)
+{
+ z_stream zs;
+ uint8_t *buf = NULL, *nbuf;
+ size_t buf_len;
+ int rc = -1, zret;
+
+ memset(&zs, 0, sizeof(zs));
+
+ zs.avail_in = *inlen;
+ zs.next_in = *inbuf;
+
+ if (inflateInit2(&zs, MAX_WBITS + 16) != Z_OK)
+ goto out;
+
+ buf_len = *inlen * 2;
+ do {
+ buf_len += GZIP_CHUNK_SIZE;
+ if ((nbuf = realloc(buf, buf_len)) == NULL)
+ err(1, NULL);
+ buf = nbuf;
+ zs.next_out = buf + zs.total_out;
+ zs.avail_out = buf_len - zs.total_out;
+
+ zret = inflate(&zs, Z_NO_FLUSH);
+ if (zret != Z_OK && zret != Z_STREAM_END)
+ goto out;
+ } while (zs.avail_out == 0);
+
+ if ((nbuf = realloc(buf, zs.total_out)) == NULL)
+ err(1, NULL);
+ buf = nbuf;
+
+ *inbuf = buf;
+ buf = NULL;
+ *inlen = zs.total_out;
+
+ rc = 0;
+ out:
+ free(buf);
+ if (inflateEnd(&zs) != Z_OK)
+ return -1;
+ return rc;
}
/*
Index: extern.h
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v
diff -u -p -r1.268 extern.h
--- extern.h 18 Nov 2025 14:04:45 -0000 1.268
+++ extern.h 30 Dec 2025 12:26:15 -0000
@@ -203,6 +203,7 @@ enum rtype {
RTYPE_GEOFEED,
RTYPE_SPL,
RTYPE_CCR,
+ RTYPE_CCR_GZ,
};
enum location {
@@ -937,9 +938,14 @@ void rrdp_fetch(unsigned int, const ch
void rrdp_abort(unsigned int);
void rrdp_http_done(unsigned int, enum http_result, const char *);
-/* Encoding functions for hex and base64. */
+
+/* File loading and decompression functions. */
unsigned char *load_file(const char *, size_t *);
+int inflate_buffer(uint8_t **, size_t *);
+
+/* Encoding functions for hex and base64. */
+
int base64_decode_len(size_t, size_t *);
int base64_decode(const unsigned char *, size_t,
unsigned char **, size_t *);
@@ -1018,12 +1024,12 @@ extern int outformats;
int outputfiles(struct validation_data *, struct stats *);
int outputheader(FILE *, struct validation_data *, struct stats *);
int output_bgpd(FILE *, struct validation_data *, struct stats *);
+int output_ccr(FILE *, struct validation_data *, struct stats *);
int output_bird(FILE *, struct validation_data *, struct stats *);
int output_csv(FILE *, struct validation_data *, struct stats *);
int output_json(FILE *, struct validation_data *, struct stats *);
int output_ometric(FILE *, struct validation_data *,
struct stats *);
-int output_ccr_der(FILE *, struct validation_data *, struct stats *);
/*
* Canonical Cache Representation
Index: filemode.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/filemode.c,v
diff -u -p -r1.73 filemode.c
--- filemode.c 5 Dec 2025 08:41:32 -0000 1.73
+++ filemode.c 30 Dec 2025 12:26:16 -0000
@@ -457,6 +457,17 @@ proc_parser_file(char *file, unsigned ch
}
}
+ type = rtype_from_file_extension(file);
+ if (type == RTYPE_CCR_GZ) {
+ if (inflate_buffer(&buf, &len) == -1) {
+ warnx("%s: gzip decompression failed", file);
+ goto out;
+ }
+ type = RTYPE_CCR;
+ }
+ if (type == RTYPE_INVALID)
+ type = rtype_from_der(file, buf, len);
+
if (!EVP_Digest(buf, len, filehash, NULL, EVP_sha256(), NULL))
errx(1, "EVP_Digest failed in %s", __func__);
@@ -472,10 +483,6 @@ proc_parser_file(char *file, unsigned ch
}
free(hash);
-
- type = rtype_from_file_extension(file);
- if (type == RTYPE_INVALID)
- type = rtype_from_der(file, buf, len);
switch (type) {
case RTYPE_ASPA:
Index: mft.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v
diff -u -p -r1.134 mft.c
--- mft.c 2 Dec 2025 06:54:08 -0000 1.134
+++ mft.c 30 Dec 2025 12:26:16 -0000
@@ -91,8 +91,12 @@ rtype_from_file_extension(const char *fn
return RTYPE_GEOFEED;
if (strcasecmp(fn + sz - 4, ".spl") == 0)
return RTYPE_SPL;
+ if (strcasecmp(fn + sz - 3, "ccr") == 0)
+ return RTYPE_CCR;
if (strcasecmp(fn + sz - 4, ".ccr") == 0)
return RTYPE_CCR;
+ if (strcasecmp(fn + sz - 6, "ccr.gz") == 0)
+ return RTYPE_CCR_GZ;
return RTYPE_INVALID;
}
Index: output.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/output.c,v
diff -u -p -r1.45 output.c
--- output.c 13 Nov 2025 15:18:53 -0000 1.45
+++ output.c 30 Dec 2025 12:26:16 -0000
@@ -70,7 +70,7 @@ static const struct outputs {
{ FORMAT_CSV, "csv", output_csv },
{ FORMAT_JSON, "json", output_json },
{ FORMAT_OMETRIC, "metrics", output_ometric },
- { FORMAT_CCR, "rpki.ccr", output_ccr_der },
+ { FORMAT_CCR, "ccr.gz", output_ccr },
{ 0, NULL, NULL }
};
Index: rpki-client.8
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/rpki-client.8,v
diff -u -p -r1.134 rpki-client.8
--- rpki-client.8 2 Dec 2025 12:21:39 -0000 1.134
+++ rpki-client.8 30 Dec 2025 12:26:16 -0000
@@ -116,11 +116,12 @@ and
.Fl -address
flags and connect with rsync-protocol locations.
.It Fl f Ar
-Attempt to decode and validate the RPKI object in
+Attempt to decode and validate the signed RPKI object or CCR in
.Ar file
using the cache stored in
.Ar cachedir
and print human-readable information about the object.
+Gzip compressed canonical cache representations are inflated on the fly.
If
.Ar file
is an rsync:// URI, the corresponding file from the cache will be used.
@@ -245,9 +246,8 @@ Defaults to
.Pp
By default
.Nm
-outputs validated payloads in
-.Fl o
-(OpenBGPD compatible) format.
+outputs validated payloads in OpenBGPD format and in compressed canonical
+cache representation format.
.Pp
.Nm
should be run hourly by
@@ -302,7 +302,7 @@ utilizes the following environment varia
URL of HTTP proxy to use.
.El
.Sh FILES
-.Bl -tag -width "/var/db/rpki-client/openbgpd" -compact
+.Bl -tag -width Ds -compact
.It Pa /etc/rpki/*.tal
default TAL files used unless
.Fl t Ar tal
@@ -320,9 +320,8 @@ is specified.
cached repository data.
.It Pa /var/db/rpki-client/openbgpd
default roa-set output file.
-.It Pa /var/db/rpki-client/rpki.ccr
-DER-encoded canonical cache representation file.
-This facility is experimental and is still subject to change.
+.It Pa /var/db/rpki-client/ccr.gz
+Gzip compressed canonical cache representation.
.El
.Sh EXIT STATUS
.Ex -std
rpki-client: compressed canonical cache representations