From: Job Snijders <job@openbsd.org>
Subject: rpki-client: expose Manifest sequence number gaps in log & telemetry
To: tech@openbsd.org
Date: Sat, 2 Nov 2024 11:56:00 +0000

Alloah,

I think it is helpful for network operators, publication point
operators, and CA operators to have more insight into whether the RP
noticed an issuance gap between two versions of a given manifest.

Detection of Manifest issuance gaps can be useful in a number of ways:

* high number of gaps all the time might be an indication the RP is not
  refreshing often enough
* the RFC 8181 publication server's ingress API endpoint has issues
* the RFC 8181 publication client has trouble reaching the server
* the CA is trying to issue manifests more than once a second
* the CA's private keys (RPKI + BPKI) are in use on a (cloned) system
* the CA's issuance database is broken

Correlation opportunities
-------------------------

Detection of a gap means some of the CA's intermediate states were
occluded from the RP; the RP operator might want to correlate this to
traffic shifts in BGP, and repository reachability issues.

The below patch emits a warning per manifest, adds metrics to the
openmetrics output, and displays a summary at the end of the run.

With tb@

OK?

Kind regards,

Job

Index: extern.h
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v
diff -u -p -r1.228 extern.h
--- extern.h	12 Sep 2024 10:33:25 -0000	1.228
+++ extern.h	2 Nov 2024 11:47:19 -0000
@@ -219,6 +219,7 @@ struct mft {
 	unsigned int	 repoid;
 	int		 talid;
 	int		 certid;
+	int		 seqnum_gap; /* was there a gap compared to prev mft? */
 };
 
 /*
@@ -584,6 +585,7 @@ enum stype {
 	STYPE_DEC_UNIQUE,
 	STYPE_PROVIDERS,
 	STYPE_OVERFLOW,
+	STYPE_SEQNUM_GAP,
 };
 
 struct repo;
@@ -598,6 +600,7 @@ struct repotalstats {
 	uint32_t	 certs; /* certificates */
 	uint32_t	 certs_fail; /* invalid certificate */
 	uint32_t	 mfts; /* total number of manifests */
+	uint32_t	 mfts_gap; /* manifests with sequence gaps */
 	uint32_t	 mfts_fail; /* failing syntactic parse */
 	uint32_t	 roas; /* route origin authorizations */
 	uint32_t	 roas_fail; /* failing syntactic parse */
@@ -690,6 +693,7 @@ struct mft	*mft_parse(X509 **, const cha
 struct mft	*mft_read(struct ibuf *);
 int		 mft_compare_issued(const struct mft *, const struct mft *);
 int		 mft_compare_seqnum(const struct mft *, const struct mft *);
+int		 mft_seqnum_gap_present(const struct mft *, const struct mft *);
 
 void		 roa_buffer(struct ibuf *, const struct roa *);
 void		 roa_free(struct roa *);
Index: filemode.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/filemode.c,v
diff -u -p -r1.49 filemode.c
--- filemode.c	20 Aug 2024 13:31:49 -0000	1.49
+++ filemode.c	2 Nov 2024 11:47:19 -0000
@@ -41,6 +41,8 @@
 #include "extern.h"
 #include "json.h"
 
+extern BN_CTX		*bn_ctx;
+
 static X509_STORE_CTX	*ctx;
 static struct auth_tree	 auths = RB_INITIALIZER(&auths);
 static struct crl_tree	 crlt = RB_INITIALIZER(&crlt);
@@ -722,6 +724,9 @@ proc_filemode(int fd)
 
 	if ((ctx = X509_STORE_CTX_new()) == NULL)
 		err(1, "X509_STORE_CTX_new");
+	if ((bn_ctx = BN_CTX_new()) == NULL)
+		err(1, "BN_CTX_new");
+
 	TAILQ_INIT(&q);
 
 	msgbuf_init(&msgq);
@@ -781,6 +786,8 @@ proc_filemode(int fd)
 	crl_tree_free(&crlt);
 
 	X509_STORE_CTX_free(ctx);
+	BN_CTX_free(bn_ctx);
+
 	ibuf_free(inbuf);
 
 	exit(0);
Index: main.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/main.c,v
diff -u -p -r1.268 main.c
--- main.c	23 Oct 2024 12:09:14 -0000	1.268
+++ main.c	2 Nov 2024 11:47:19 -0000
@@ -639,6 +639,8 @@ entity_process(struct ibuf *b, struct st
 			break;
 		}
 		mft = mft_read(b);
+		if (mft->seqnum_gap)
+			repo_stat_inc(rp, talid, type, STYPE_SEQNUM_GAP);
 		queue_add_from_mft(mft);
 		mft_free(mft);
 		break;
@@ -764,6 +766,7 @@ sum_stats(const struct repo *rp, const s
 
 	out->mfts += in->mfts;
 	out->mfts_fail += in->mfts_fail;
+	out->mfts_gap += in->mfts_gap;
 	out->certs += in->certs;
 	out->certs_fail += in->certs_fail;
 	out->roas += in->roas;
@@ -1500,8 +1503,9 @@ main(int argc, char *argv[])
 	    stats.repo_tal_stats.certs, stats.repo_tal_stats.certs_fail);
 	printf("Trust Anchor Locators: %u (%u invalid)\n",
 	    stats.tals, talsz - stats.tals);
-	printf("Manifests: %u (%u failed parse)\n",
-	    stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail);
+	printf("Manifests: %u (%u failed parse, %u seqnum gaps)\n",
+	    stats.repo_tal_stats.mfts, stats.repo_tal_stats.mfts_fail,
+	    stats.repo_tal_stats.mfts_gap);
 	printf("Certificate revocation lists: %u\n", stats.repo_tal_stats.crls);
 	printf("Ghostbuster records: %u\n", stats.repo_tal_stats.gbrs);
 	printf("Trust Anchor Keys: %u\n", stats.repo_tal_stats.taks);
Index: mft.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v
diff -u -p -r1.119 mft.c
--- mft.c	12 Sep 2024 10:33:25 -0000	1.119
+++ mft.c	2 Nov 2024 11:47:19 -0000
@@ -35,6 +35,7 @@
 #include "extern.h"
 
 extern ASN1_OBJECT	*mft_oid;
+BN_CTX			*bn_ctx;
 
 /*
  * Types and templates for the Manifest eContent, RFC 6486, section 4.2.
@@ -538,6 +539,7 @@ mft_buffer(struct ibuf *b, const struct 
 	io_simple_buffer(b, &p->repoid, sizeof(p->repoid));
 	io_simple_buffer(b, &p->talid, sizeof(p->talid));
 	io_simple_buffer(b, &p->certid, sizeof(p->certid));
+	io_simple_buffer(b, &p->seqnum_gap, sizeof(p->seqnum_gap));
 	io_str_buffer(b, p->path);
 
 	io_str_buffer(b, p->aia);
@@ -571,6 +573,7 @@ mft_read(struct ibuf *b)
 	io_read_buf(b, &p->repoid, sizeof(p->repoid));
 	io_read_buf(b, &p->talid, sizeof(p->talid));
 	io_read_buf(b, &p->certid, sizeof(p->certid));
+	io_read_buf(b, &p->seqnum_gap, sizeof(p->seqnum_gap));
 	io_read_str(b, &p->path);
 
 	io_read_str(b, &p->aia);
@@ -627,4 +630,36 @@ mft_compare_seqnum(const struct mft *a, 
 		return -1;
 
 	return 0;
+}
+
+/*
+ * Test if there is a gap in the sequence numbers of two MFTs.
+ * Return 1 if a gap is detected.
+ */
+int
+mft_seqnum_gap_present(const struct mft *a, const struct mft *b)
+{
+	BIGNUM *diff, *seqnum_a, *seqnum_b;
+	int ret = 0;
+
+	BN_CTX_start(bn_ctx);
+	if ((diff = BN_CTX_get(bn_ctx)) == NULL ||
+	    (seqnum_a = BN_CTX_get(bn_ctx)) == NULL ||
+	    (seqnum_b = BN_CTX_get(bn_ctx)) == NULL)
+		errx(1, "BN_CTX_get");
+
+	if (!BN_hex2bn(&seqnum_a, a->seqnum))
+		errx(1, "BN_hex2bn");
+
+	if (!BN_hex2bn(&seqnum_b, b->seqnum))
+		errx(1, "BN_hex2bn");
+
+	if (!BN_sub(diff, seqnum_a, seqnum_b))
+		errx(1, "BN_sub");
+
+	ret = !BN_is_one(diff);
+
+	BN_CTX_end(bn_ctx);
+
+	return ret;
 }
Index: output-ometric.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/output-ometric.c,v
diff -u -p -r1.11 output-ometric.c
--- output-ometric.c	3 Sep 2024 15:04:48 -0000	1.11
+++ output-ometric.c	2 Nov 2024 11:47:19 -0000
@@ -47,6 +47,8 @@ set_common_stats(const struct repotalsta
 	    OKV("type", "state"), OKV("manifest", "valid"), ol);
 	ometric_set_int_with_labels(metric, in->mfts_fail,
 	    OKV("type", "state"), OKV("manifest", "failed parse"), ol);
+	ometric_set_int_with_labels(metric, in->mfts_gap,
+	    OKV("type", "state"), OKV("manifest", "sequence gap"), ol);
 
 	ometric_set_int_with_labels(metric, in->roas,
 	    OKV("type", "state"), OKV("roa", "valid"), ol);
Index: parser.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/parser.c,v
diff -u -p -r1.143 parser.c
--- parser.c	29 Aug 2024 13:46:28 -0000	1.143
+++ parser.c	2 Nov 2024 11:47:19 -0000
@@ -40,6 +40,8 @@
 
 extern int certid;
 
+extern BN_CTX		*bn_ctx;
+
 static X509_STORE_CTX	*ctx;
 static struct auth_tree	 auths = RB_INITIALIZER(&auths);
 static struct crl_tree	 crlt = RB_INITIALIZER(&crlt);
@@ -453,6 +455,14 @@ proc_parser_mft_pre(struct entity *entp,
 		goto err;
 	}
 
+	if (seqnum_cmp > 0) {
+		if (mft_seqnum_gap_present(mft, cached_mft)) {
+			mft->seqnum_gap = 1;
+			warnx("%s: seqnum gap detected #%s -> #%s", file,
+			    cached_mft->seqnum, mft->seqnum);
+		}
+	}
+
 	return mft;
 
  err:
@@ -1055,6 +1065,8 @@ proc_parser(int fd)
 
 	if ((ctx = X509_STORE_CTX_new()) == NULL)
 		err(1, "X509_STORE_CTX_new");
+	if ((bn_ctx = BN_CTX_new()) == NULL)
+		err(1, "BN_CTX_new");
 
 	TAILQ_INIT(&q);
 
@@ -1114,6 +1126,8 @@ proc_parser(int fd)
 	crl_tree_free(&crlt);
 
 	X509_STORE_CTX_free(ctx);
+	BN_CTX_free(bn_ctx);
+
 	msgbuf_clear(&msgq);
 
 	ibuf_free(inbuf);
Index: repo.c
===================================================================
RCS file: /cvs/src/usr.sbin/rpki-client/repo.c,v
diff -u -p -r1.68 repo.c
--- repo.c	27 Sep 2024 12:55:03 -0000	1.68
+++ repo.c	2 Nov 2024 11:47:20 -0000
@@ -1500,6 +1500,8 @@ repo_stat_inc(struct repo *rp, int talid
 			rp->stats[talid].mfts++;
 		if (subtype == STYPE_FAIL)
 			rp->stats[talid].mfts_fail++;
+		if (subtype == STYPE_SEQNUM_GAP)
+			rp->stats[talid].mfts_gap++;
 		break;
 	case RTYPE_ROA:
 		switch (subtype) {