Download raw body.
rpki-client: expose Manifest sequence number gaps in log & telemetry
Alloah,
I think it is helpful for network operators, publication point
operators, and CA operators to have more insight into whether the RP
noticed an issuance gap between two versions of a given manifest.
Detection of Manifest issuance gaps can be useful in a number of ways:
* high number of gaps all the time might be an indication the RP is not
refreshing often enough
* the RFC 8181 publication server's ingress API endpoint has issues
* the RFC 8181 publication client has trouble reaching the server
* the CA is trying to issue manifests more than once a second
* the CA's private keys (RPKI + BPKI) are in use on a (cloned) system
* the CA's issuance database is broken
Correlation opportunities
-------------------------
Detection of a gap means some of the CA's intermediate states were
occluded from the RP; the RP operator might want to correlate this to
traffic shifts in BGP, and repository reachability issues.
The below patch emits a warning per manifest, adds metrics to the
openmetrics output, and displays a summary at the end of the run.
With tb@
OK?
Kind regards,
Job
Index: extern.h
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v
diff -u -p -r1.228 extern.h
--- extern.h 12 Sep 2024 10:33:25 -0000 1.228
+++ extern.h 2 Nov 2024 11:47:19 -0000
@@ -219,6 +219,7 @@ struct mft {
unsigned int repoid;
int talid;
int certid;
+ int seqnum_gap; /* was there a gap compared to prev mft? */
};
/*
@@ -584,6 +585,7 @@ enum stype {
STYPE_DEC_UNIQUE,
STYPE_PROVIDERS,
STYPE_OVERFLOW,
+ STYPE_SEQNUM_GAP,
};
struct repo;
@@ -598,6 +600,7 @@ struct repotalstats {
uint32_t certs; /* certificates */
uint32_t certs_fail; /* invalid certificate */
uint32_t mfts; /* total number of manifests */
+ uint32_t mfts_gap; /* manifests with sequence gaps */
uint32_t mfts_fail; /* failing syntactic parse */
uint32_t roas; /* route origin authorizations */
uint32_t roas_fail; /* failing syntactic parse */
@@ -690,6 +693,7 @@ struct mft *mft_parse(X509 **, const cha
struct mft *mft_read(struct ibuf *);
int mft_compare_issued(const struct mft *, const struct mft *);
int mft_compare_seqnum(const struct mft *, const struct mft *);
+int mft_seqnum_gap_present(const struct mft *, const struct mft *);
void roa_buffer(struct ibuf *, const struct roa *);
void roa_free(struct roa *);
Index: filemode.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/filemode.c,v
diff -u -p -r1.49 filemode.c
--- filemode.c 20 Aug 2024 13:31:49 -0000 1.49
+++ filemode.c 2 Nov 2024 11:47:19 -0000
@@ -41,6 +41,8 @@
#include "extern.h"
#include "json.h"
+extern BN_CTX *bn_ctx;
+
static X509_STORE_CTX *ctx;
static struct auth_tree auths = RB_INITIALIZER(&auths);
static struct crl_tree crlt = RB_INITIALIZER(&crlt);
@@ -722,6 +724,9 @@ proc_filemode(int fd)
if ((ctx = X509_STORE_CTX_new()) == NULL)
err(1, "X509_STORE_CTX_new");
+ if ((bn_ctx = BN_CTX_new()) == NULL)
+ err(1, "BN_CTX_new");
+
TAILQ_INIT(&q);
msgbuf_init(&msgq);
@@ -781,6 +786,8 @@ proc_filemode(int fd)
crl_tree_free(&crlt);
X509_STORE_CTX_free(ctx);
+ BN_CTX_free(bn_ctx);
+
ibuf_free(inbuf);
exit(0);
Index: main.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/main.c,v
diff -u -p -r1.268 main.c
--- main.c 23 Oct 2024 12:09:14 -0000 1.268
+++ main.c 2 Nov 2024 11:47:19 -0000
@@ -639,6 +639,8 @@ entity_process(struct ibuf *b, struct st
break;
}
mft = mft_read(b);
+ if (mft->seqnum_gap)
+ repo_stat_inc(rp, talid, type, STYPE_SEQNUM_GAP);
queue_add_from_mft(mft);
mft_free(mft);
break;
@@ -764,6 +766,7 @@ sum_stats(const struct repo *rp, const s
out->mfts += in->mfts;
out->mfts_fail += in->mfts_fail;
+ out->mfts_gap += in->mfts_gap;
out->certs += in->certs;
out->certs_fail += in->certs_fail;
out->roas += in->roas;
@@ -1500,8 +1503,9 @@ main(int argc, char *argv[])
stats.repo_tal_stats.certs, stats.repo_tal_stats.certs_fail);
printf("Trust Anchor Locators: %u (%u invalid)\n",
stats.tals, talsz - stats.tals);
- printf("Manifests: %u (%u failed parse)\n",
- stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail);
+ printf("Manifests: %u (%u failed parse, %u seqnum gaps)\n",
+ stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail,
+ stats.repo_tal_stats.mfts_gap);
printf("Certificate revocation lists: %u\n", stats.repo_tal_stats.crls);
printf("Ghostbuster records: %u\n", stats.repo_tal_stats.gbrs);
printf("Trust Anchor Keys: %u\n", stats.repo_tal_stats.taks);
Index: mft.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v
diff -u -p -r1.119 mft.c
--- mft.c 12 Sep 2024 10:33:25 -0000 1.119
+++ mft.c 2 Nov 2024 11:47:19 -0000
@@ -35,6 +35,7 @@
#include "extern.h"
extern ASN1_OBJECT *mft_oid;
+BN_CTX *bn_ctx;
/*
* Types and templates for the Manifest eContent, RFC 6486, section 4.2.
@@ -538,6 +539,7 @@ mft_buffer(struct ibuf *b, const struct
io_simple_buffer(b, &p->repoid, sizeof(p->repoid));
io_simple_buffer(b, &p->talid, sizeof(p->talid));
io_simple_buffer(b, &p->certid, sizeof(p->certid));
+ io_simple_buffer(b, &p->seqnum_gap, sizeof(p->seqnum_gap));
io_str_buffer(b, p->path);
io_str_buffer(b, p->aia);
@@ -571,6 +573,7 @@ mft_read(struct ibuf *b)
io_read_buf(b, &p->repoid, sizeof(p->repoid));
io_read_buf(b, &p->talid, sizeof(p->talid));
io_read_buf(b, &p->certid, sizeof(p->certid));
+ io_read_buf(b, &p->seqnum_gap, sizeof(p->seqnum_gap));
io_read_str(b, &p->path);
io_read_str(b, &p->aia);
@@ -627,4 +630,36 @@ mft_compare_seqnum(const struct mft *a,
return -1;
return 0;
+}
+
+/*
+ * Test if there is a gap in the sequence numbers of two MFTs.
+ * Return 1 if a gap is detected.
+ */
+int
+mft_seqnum_gap_present(const struct mft *a, const struct mft *b)
+{
+ BIGNUM *diff, *seqnum_a, *seqnum_b;
+ int ret = 0;
+
+ BN_CTX_start(bn_ctx);
+ if ((diff = BN_CTX_get(bn_ctx)) == NULL ||
+ (seqnum_a = BN_CTX_get(bn_ctx)) == NULL ||
+ (seqnum_b = BN_CTX_get(bn_ctx)) == NULL)
+ errx(1, "BN_CTX_get");
+
+ if (!BN_hex2bn(&seqnum_a, a->seqnum))
+ errx(1, "BN_hex2bn");
+
+ if (!BN_hex2bn(&seqnum_b, b->seqnum))
+ errx(1, "BN_hex2bn");
+
+ if (!BN_sub(diff, seqnum_a, seqnum_b))
+ errx(1, "BN_sub");
+
+ ret = !BN_is_one(diff);
+
+ BN_CTX_end(bn_ctx);
+
+ return ret;
}
Index: output-ometric.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/output-ometric.c,v
diff -u -p -r1.11 output-ometric.c
--- output-ometric.c 3 Sep 2024 15:04:48 -0000 1.11
+++ output-ometric.c 2 Nov 2024 11:47:19 -0000
@@ -47,6 +47,8 @@ set_common_stats(const struct repotalsta
OKV("type", "state"), OKV("manifest", "valid"), ol);
ometric_set_int_with_labels(metric, in->mfts_fail,
OKV("type", "state"), OKV("manifest", "failed parse"), ol);
+ ometric_set_int_with_labels(metric, in->mfts_gap,
+ OKV("type", "state"), OKV("manifest", "sequence gap"), ol);
ometric_set_int_with_labels(metric, in->roas,
OKV("type", "state"), OKV("roa", "valid"), ol);
Index: parser.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/parser.c,v
diff -u -p -r1.143 parser.c
--- parser.c 29 Aug 2024 13:46:28 -0000 1.143
+++ parser.c 2 Nov 2024 11:47:19 -0000
@@ -40,6 +40,8 @@
extern int certid;
+extern BN_CTX *bn_ctx;
+
static X509_STORE_CTX *ctx;
static struct auth_tree auths = RB_INITIALIZER(&auths);
static struct crl_tree crlt = RB_INITIALIZER(&crlt);
@@ -453,6 +455,14 @@ proc_parser_mft_pre(struct entity *entp,
goto err;
}
+ if (seqnum_cmp > 0) {
+ if (mft_seqnum_gap_present(mft, cached_mft)) {
+ mft->seqnum_gap = 1;
+ warnx("%s: seqnum gap detected #%s -> #%s", file,
+ cached_mft->seqnum, mft->seqnum);
+ }
+ }
+
return mft;
err:
@@ -1055,6 +1065,8 @@ proc_parser(int fd)
if ((ctx = X509_STORE_CTX_new()) == NULL)
err(1, "X509_STORE_CTX_new");
+ if ((bn_ctx = BN_CTX_new()) == NULL)
+ err(1, "BN_CTX_new");
TAILQ_INIT(&q);
@@ -1114,6 +1126,8 @@ proc_parser(int fd)
crl_tree_free(&crlt);
X509_STORE_CTX_free(ctx);
+ BN_CTX_free(bn_ctx);
+
msgbuf_clear(&msgq);
ibuf_free(inbuf);
Index: repo.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/repo.c,v
diff -u -p -r1.68 repo.c
--- repo.c 27 Sep 2024 12:55:03 -0000 1.68
+++ repo.c 2 Nov 2024 11:47:20 -0000
@@ -1500,6 +1500,8 @@ repo_stat_inc(struct repo *rp, int talid
rp->stats[talid].mfts++;
if (subtype == STYPE_FAIL)
rp->stats[talid].mfts_fail++;
+ if (subtype == STYPE_SEQNUM_GAP)
+ rp->stats[talid].mfts_gap++;
break;
case RTYPE_ROA:
switch (subtype) {
rpki-client: expose Manifest sequence number gaps in log & telemetry