From 16f94533ef453314fa6d520e49d1a23a36fd8003 Mon Sep 17 00:00:00 2001 From: job Date: Fri, 16 Jan 2026 11:25:27 +0000 Subject: [PATCH] Inflate gzip compressed CCR files on the fly in filemode Turns out CCR data is highly compressable (~50% reduction with gzip). Filemode recognizes compressed files by the .gz filename extension and handles those transparently, i.e. 'rpki-client -jf *.ccr.gz *.mft.gz' will output the hash identifier for a given file's uncompressed form. OK tb@ --- regress/usr.sbin/rpki-client/Makefile.inc | 6 +-- usr.sbin/rpki-client/encoding.c | 57 ++++++++++++++++++++++- usr.sbin/rpki-client/extern.h | 10 ++-- usr.sbin/rpki-client/filemode.c | 20 +++++++- usr.sbin/rpki-client/mft.c | 4 +- usr.sbin/rpki-client/rpki-client.8 | 12 ++--- 6 files changed, 94 insertions(+), 15 deletions(-) diff --git a/regress/usr.sbin/rpki-client/Makefile.inc b/regress/usr.sbin/rpki-client/Makefile.inc index 6e576cdfb43..d9dac17950b 100644 --- a/regress/usr.sbin/rpki-client/Makefile.inc +++ b/regress/usr.sbin/rpki-client/Makefile.inc @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile.inc,v 1.45 2026/01/13 21:36:17 job Exp $ +# $OpenBSD: Makefile.inc,v 1.46 2026/01/16 11:25:27 job Exp $ .PATH: ${.CURDIR}/../../../../usr.sbin/rpki-client @@ -22,8 +22,8 @@ REGRESS_TARGETS += run-regress-$p .endfor CFLAGS+= -I${.CURDIR}/.. -I${.CURDIR}/../../../../usr.sbin/rpki-client -LDADD+= -lcrypto -lutil -lpthread -DPADD+= ${LIBCRYPTO} ${LIBUTIL} ${LIBPTHREAD} +LDADD+= -lcrypto -lutil -lpthread -lz +DPADD+= ${LIBCRYPTO} ${LIBUTIL} ${LIBPTHREAD} ${LIBZ} CLEANFILES+= *.out *.err *.txt diff --git a/usr.sbin/rpki-client/encoding.c b/usr.sbin/rpki-client/encoding.c index 96ac2baf076..4484da26b25 100644 --- a/usr.sbin/rpki-client/encoding.c +++ b/usr.sbin/rpki-client/encoding.c @@ -1,4 +1,4 @@ -/* $OpenBSD: encoding.c,v 1.14 2025/09/09 08:23:24 job Exp $ */ +/* $OpenBSD: encoding.c,v 1.15 2026/01/16 11:25:27 job Exp $ */ /* * Copyright (c) 2020 Claudio Jeker * @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -73,6 +74,60 @@ err: return NULL; } +#define GZIP_CHUNK_SIZE (32 * 1024) + +/* + * One-shot gzip data decompressor. + * On success return the inflated object, or NULL on error. + * Caller must free the newly allocated object. + */ +unsigned char * +inflate_buffer(uint8_t *inbuf, size_t inlen, size_t *outlen) +{ + z_stream zs; + uint8_t *buf = NULL, *nbuf; + size_t buf_len; + int zret; + + memset(&zs, 0, sizeof(zs)); + + zs.avail_in = inlen; + zs.next_in = inbuf; + + if (inflateInit2(&zs, MAX_WBITS + 16) != Z_OK) + goto err; + + buf_len = inlen * 2; + do { + buf_len += GZIP_CHUNK_SIZE; + if ((nbuf = realloc(buf, buf_len)) == NULL) + err(1, NULL); + buf = nbuf; + zs.next_out = buf + zs.total_out; + zs.avail_out = buf_len - zs.total_out; + + zret = inflate(&zs, Z_NO_FLUSH); + if (zret != Z_OK && zret != Z_STREAM_END) + goto err; + } while (zs.avail_out == 0); + + if (inflateEnd(&zs) != Z_OK) + goto err; + + /* shrink to right size */ + if ((nbuf = realloc(buf, zs.total_out)) == NULL) + err(1, NULL); + buf = nbuf; + + *outlen = zs.total_out; + return buf; + + err: + inflateEnd(&zs); + free(buf); + return NULL; +} + /* * Return the size of the data blob in outlen for an inlen sized base64 buffer. * Returns 0 on success and -1 if inlen would overflow an int. diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h index 7e8cfe73c1d..fcb638a9833 100644 --- a/usr.sbin/rpki-client/extern.h +++ b/usr.sbin/rpki-client/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.269 2026/01/13 21:36:17 job Exp $ */ +/* $OpenBSD: extern.h,v 1.270 2026/01/16 11:25:27 job Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -202,6 +202,7 @@ enum rtype { RTYPE_TAK, RTYPE_SPL, RTYPE_CCR, + RTYPE_GZ, }; enum location { @@ -911,9 +912,12 @@ void rrdp_fetch(unsigned int, const char *, const char *, void rrdp_abort(unsigned int); void rrdp_http_done(unsigned int, enum http_result, const char *); -/* Encoding functions for hex and base64. */ +/* File loading and decompression functions. */ unsigned char *load_file(const char *, size_t *); +unsigned char *inflate_buffer(uint8_t *, size_t, size_t *); + +/* Encoding functions for hex and base64. */ int base64_decode_len(size_t, size_t *); int base64_decode(const unsigned char *, size_t, unsigned char **, size_t *); @@ -992,11 +996,11 @@ int outputfiles(struct validation_data *, struct stats *); int outputheader(FILE *, struct validation_data *, struct stats *); int output_bgpd(FILE *, struct validation_data *, struct stats *); int output_bird(FILE *, struct validation_data *, struct stats *); +int output_ccr_der(FILE *, struct validation_data *, struct stats *); int output_csv(FILE *, struct validation_data *, struct stats *); int output_json(FILE *, struct validation_data *, struct stats *); int output_ometric(FILE *, struct validation_data *, struct stats *); -int output_ccr_der(FILE *, struct validation_data *, struct stats *); /* * Canonical Cache Representation diff --git a/usr.sbin/rpki-client/filemode.c b/usr.sbin/rpki-client/filemode.c index 6c42d9090ac..e85137c41db 100644 --- a/usr.sbin/rpki-client/filemode.c +++ b/usr.sbin/rpki-client/filemode.c @@ -1,4 +1,4 @@ -/* $OpenBSD: filemode.c,v 1.75 2026/01/13 21:36:17 job Exp $ */ +/* $OpenBSD: filemode.c,v 1.76 2026/01/16 11:25:27 job Exp $ */ /* * Copyright (c) 2019 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons @@ -457,6 +457,24 @@ proc_parser_file(char *file, unsigned char *in_buf, size_t len) } } + if (rtype_from_file_extension(file) == RTYPE_GZ) { + size_t full_len; + char *gz_ext; + + if ((buf = inflate_buffer(buf, len, &full_len)) == NULL) { + warnx("%s: gzip decompression failed", file); + goto out; + } + len = full_len; + + /* zap trailing .gz */ + if ((gz_ext = strrchr(file, '.')) == NULL) { + warnx("%s: unreachable: missing . in filename?", file); + goto out; + } + *gz_ext = '\0'; + } + if (!EVP_Digest(buf, len, filehash, NULL, EVP_sha256(), NULL)) errx(1, "EVP_Digest failed in %s", __func__); diff --git a/usr.sbin/rpki-client/mft.c b/usr.sbin/rpki-client/mft.c index 3be606003e6..a4dc446f544 100644 --- a/usr.sbin/rpki-client/mft.c +++ b/usr.sbin/rpki-client/mft.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mft.c,v 1.135 2026/01/13 21:36:17 job Exp $ */ +/* $OpenBSD: mft.c,v 1.136 2026/01/16 11:25:27 job Exp $ */ /* * Copyright (c) 2022 Theo Buehler * Copyright (c) 2019 Kristaps Dzonsons @@ -91,6 +91,8 @@ rtype_from_file_extension(const char *fn) return RTYPE_SPL; if (strcasecmp(fn + sz - 4, ".ccr") == 0) return RTYPE_CCR; + if (strcasecmp(fn + sz - 3, ".gz") == 0) + return RTYPE_GZ; return RTYPE_INVALID; } diff --git a/usr.sbin/rpki-client/rpki-client.8 b/usr.sbin/rpki-client/rpki-client.8 index 1eb51bb84e1..bad40347d68 100644 --- a/usr.sbin/rpki-client/rpki-client.8 +++ b/usr.sbin/rpki-client/rpki-client.8 @@ -1,4 +1,4 @@ -.\" $OpenBSD: rpki-client.8,v 1.135 2026/01/13 21:36:17 job Exp $ +.\" $OpenBSD: rpki-client.8,v 1.136 2026/01/16 11:25:27 job Exp $ .\" .\" Copyright (c) 2019 Kristaps Dzonsons .\" @@ -14,7 +14,7 @@ .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. .\" -.Dd $Mdocdate: January 13 2026 $ +.Dd $Mdocdate: January 16 2026 $ .Dt RPKI-CLIENT 8 .Os .Sh NAME @@ -116,11 +116,12 @@ and .Fl -address flags and connect with rsync-protocol locations. .It Fl f Ar -Attempt to decode and validate the RPKI object in +Attempt to decode and validate the signed RPKI object or CCR in .Ar file using the cache stored in .Ar cachedir and print human-readable information about the object. +Gzip compressed files are inflated on the fly. If .Ar file is an rsync:// URI, the corresponding file from the cache will be used. @@ -245,9 +246,8 @@ Defaults to .Pp By default .Nm -outputs validated payloads in -.Fl o -(OpenBGPD compatible) format. +outputs validated payloads in OpenBGPD format and in canonical cache +representation format. .Pp .Nm should be run hourly by