From 67c9465ace456769bb2a41b72ade07c5c46953d6 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Wed, 29 Sep 2021 18:52:35 -0400 Subject: [PATCH] Add multi-threaded compression support Note it is only supported for txz format for now. However, we can easily extend it to support tzst format when libarchive supports it. --- docs/pkg-create.8 | 15 +++++++++++++++ docs/pkg.conf.5 | 3 +++ docs/pkg_create.3 | 15 +++++++++++++-- libpkg/libpkg.ver | 1 + libpkg/packing.c | 19 +++++++++++++++---- libpkg/pkg.h.in | 1 + libpkg/pkg_config.c | 8 ++++++++ libpkg/pkg_create.c | 10 +++++++++- libpkg/pkg_repo_create.c | 2 +- libpkg/private/pkg.h | 6 +++++- src/create.c | 21 ++++++++++++++++----- 11 files changed, 87 insertions(+), 14 deletions(-) diff --git a/docs/pkg-create.8 b/docs/pkg-create.8 index af3837377c..bb207e1bfd 100644 --- a/docs/pkg-create.8 +++ b/docs/pkg-create.8 @@ -25,6 +25,7 @@ .Sh SYNOPSIS .Nm .Op Fl enqv +.Op Fl T Ar threads .Op Fl f Ar format .Op Fl l Ar level .Op Fl o Ar outdir @@ -34,6 +35,7 @@ .Fl m Ar metadatadir .Nm .Op Fl enqv +.Op Fl T Ar threads .Op Fl f Ar format .Op Fl l Ar level .Op Fl o Ar outdir @@ -42,6 +44,7 @@ .Fl M Ar manifest .Nm .Op Fl egnqvx +.Op Fl T Ar threads .Op Fl f Ar format .Op Fl l Ar level .Op Fl o Ar outdir @@ -50,6 +53,7 @@ .Ar pkg-name ... .Nm .Op Fl enqv +.Op Fl T Ar threads .Op Fl f Ar format .Op Fl l Ar level .Op Fl o Ar outdir @@ -59,6 +63,7 @@ .\" --------------------------------------------------------------------------- .Pp .Nm +.Op Cm --threads Ar threads .Op Cm --expand-manifest .Op Cm --no-clobber .Op Cm --quiet @@ -70,6 +75,7 @@ .Op Cm --root-dir Ar rootdir .Cm --metadata Ar metadatadir .Nm +.Op Cm --threads Ar threads .Op Cm --expand-manifest .Op Cm --no-clobber .Op Cm --quiet @@ -80,6 +86,7 @@ .Op Cm --root-dir Ar rootdir .Cm --manifest Ar manifest .Nm +.Op Cm --threads Ar threads .Op Cm --expand-manifest .Op Cm --{glob,no-clobber,regex} .Op Cm --quiet @@ -90,6 +97,7 @@ .Op Cm --root-dir Ar rootdir .Ar pkg-name ... .Nm +.Op Cm --threads Ar threads .Op Cm --expand-manifest .Op Cm --no-clobber .Op Cm --quiet @@ -130,6 +138,13 @@ command. The following options are supported by .Nm : .Bl -tag -width ".Fl m Ar metadatadir" +.It Fl T Ar threads , Cm --threads Ar threads +Set the number of worker +.Ar threads +to be used when packing. +It can be any valid non-negative numeric. +It may be 0 if the underlying compression can detect the number of available +cores and the user wants to use the value. .It Fl a , Cm --all Create package tarballs from all packages installed on your system. This option is incompatible with the diff --git a/docs/pkg.conf.5 b/docs/pkg.conf.5 index 581a183033..3a61ae9609 100644 --- a/docs/pkg.conf.5 +++ b/docs/pkg.conf.5 @@ -122,6 +122,9 @@ Set the default compression level, special values are: .It 0 default value per libarchive developers .It -1 default value per pkg developers (default) .El +.It Cm COMPRESSION_THREADS: integer +Set the default number of compression threads. +Default: 1. .It Cm CONSERVATIVE_UPGRADE: boolean Ensure in multi repository mode that the priority is given as much as possible to the repository where a package was first installed from. diff --git a/docs/pkg_create.3 b/docs/pkg_create.3 index 1a045e319e..bd5a34b6ad 100644 --- a/docs/pkg_create.3 +++ b/docs/pkg_create.3 @@ -5,8 +5,8 @@ .Nm pkg_create , pkg_create_i , .Nm pkg_create_new , pkg_create_free , .Nm pkg_create_set_format , pkg_create_set_overwrite , -.Nm pkg_create_set_compression_level , pkg_set_rootdir , pkg_set_output_dir , -.Nm pkg_create_set_timestamp +.Nm pkg_create_set_compression_level , pkg_create_set_compression_threads , +.Nm pkg_set_rootdir , pkg_set_output_dir , pkg_create_set_timestamp .Nd create packages .Sh LIBRARY .Lb libpkg @@ -21,6 +21,8 @@ .Ft void .Fn pkg_create_set_compression_level "struct pkg_create *" "int" .Ft void +.Fn pkg_create_set_compression_threads "struct pkg_create *" "unsigned" +.Ft void .Fn pkg_create_set_overwrite "struct pkg_create *" "bool" .Ft void .Fn pkg_create_set_rootdir "struct pkg_create *" "const char *" @@ -63,6 +65,15 @@ Set the default (as specified in libarchive) Set the best compression ratio .El .Pp +.Fn pkg_create_set_compression_threads +take an +.Ft unsigned +argument which represents the number of worker threads. +a special value is accepted: +.Bl -tag -width indentation +.It Va 0 +Set the number of available cores (as specified in libarchive) +.Pp .Fn pkg_create_set_overwrite Accept Va a boolean to define the default behaviour when creating a package and a local file already exists. diff --git a/libpkg/libpkg.ver b/libpkg/libpkg.ver index 61a6d23b59..e61beb6d56 100644 --- a/libpkg/libpkg.ver +++ b/libpkg/libpkg.ver @@ -31,6 +31,7 @@ global: pkg_create_new; pkg_create_repo; pkg_create_set_compression_level; + pkg_create_set_compression_threads; pkg_create_set_format; pkg_create_set_output_dir; pkg_create_set_overwrite; diff --git a/libpkg/packing.c b/libpkg/packing.c index 816131551e..e5def2b706 100644 --- a/libpkg/packing.c +++ b/libpkg/packing.c @@ -41,7 +41,7 @@ #include "private/event.h" #include "private/pkg.h" -static const char *packing_set_format(struct archive *a, pkg_formats format, int clevel); +static const char *packing_set_format(struct archive *a, pkg_formats format, int clevel, unsigned cthreads); struct packing { struct archive *aread; @@ -52,7 +52,7 @@ struct packing { int packing_init(struct packing **pack, const char *path, pkg_formats format, int clevel, - time_t timestamp, bool overwrite, bool compat_symlink) + unsigned cthreads, time_t timestamp, bool overwrite, bool compat_symlink) { char archive_path[MAXPATHLEN]; char archive_symlink[MAXPATHLEN]; @@ -84,7 +84,7 @@ packing_init(struct packing **pack, const char *path, pkg_formats format, int cl (*pack)->awrite = archive_write_new(); archive_write_set_format_pax_restricted((*pack)->awrite); - ext = packing_set_format((*pack)->awrite, format, clevel); + ext = packing_set_format((*pack)->awrite, format, clevel, cthreads); if (ext == NULL) { archive_read_close((*pack)->aread); archive_read_free((*pack)->aread); @@ -348,7 +348,7 @@ packing_finish(struct packing *pack) } static const char * -packing_set_format(struct archive *a, pkg_formats format, int clevel) +packing_set_format(struct archive *a, pkg_formats format, int clevel, unsigned cthreads) { const char *notsupp_fmt = "%s is not supported, trying %s"; @@ -446,6 +446,17 @@ packing_set_format(struct archive *a, pkg_formats format, int clevel) pkg_emit_error("bad compression-level %d", clevel); } + if (format != TXZ && cthreads != 1) + pkg_emit_error("Multithreaded compression only makes sense with txz"); + + if (elected_format == TXZ && cthreads != 1) { + char buf[16]; + + snprintf(buf, sizeof(buf), "%u", cthreads); + if (archive_write_set_filter_option(a, NULL, "threads", buf) != ARCHIVE_OK) + pkg_emit_error("bad threads %u", cthreads); + } + return (packing_format_to_string(elected_format)); } diff --git a/libpkg/pkg.h.in b/libpkg/pkg.h.in index 5ff73dcc5b..26b6d73dca 100644 --- a/libpkg/pkg.h.in +++ b/libpkg/pkg.h.in @@ -1648,6 +1648,7 @@ struct pkg_create *pkg_create_new(void); void pkg_create_free(struct pkg_create *); bool pkg_create_set_format(struct pkg_create *, const char *); void pkg_create_set_compression_level(struct pkg_create *, int); +void pkg_create_set_compression_threads(struct pkg_create *, unsigned); void pkg_create_set_overwrite(struct pkg_create *, bool); void pkg_create_set_rootdir(struct pkg_create *, const char *); void pkg_create_set_output_dir(struct pkg_create *, const char *); diff --git a/libpkg/pkg_config.c b/libpkg/pkg_config.c index 4c9e9f4e18..c6c1a381bc 100644 --- a/libpkg/pkg_config.c +++ b/libpkg/pkg_config.c @@ -76,6 +76,7 @@ struct pkg_ctx ctx = { .backup_libraries = false, .triggers = true, .compression_level = -1, + .compression_threads = 1, .defer_triggers = false, }; @@ -479,6 +480,12 @@ static struct config_entry c[] = { "-1", "Set the default compression level", }, + { + PKG_INT, + "COMPRESSION_THREADS", + "1", + "Set the default number of compression threads", + }, { PKG_BOOL, "ARCHIVE_SYMLINK", @@ -1334,6 +1341,7 @@ pkg_ini(const char *path, const char *reposdir, pkg_init_flags flags) ctx.triggers = pkg_object_bool(pkg_config_get("PKG_TRIGGERS_ENABLE")); ctx.triggers_path = pkg_object_string(pkg_config_get("PKG_TRIGGERS_DIR")); ctx.compression_level = pkg_object_int(pkg_config_get("COMPRESSION_LEVEL")); + ctx.compression_threads = pkg_object_int(pkg_config_get("COMPRESSION_THREADS")); ctx.archive_symlink = pkg_object_bool(pkg_config_get("ARCHIVE_SYMLINK")); ctx.repo_accept_legacy_pkg = pkg_object_bool(pkg_config_get("REPO_ACCEPT_LEGACY_PKG")); diff --git a/libpkg/pkg_create.c b/libpkg/pkg_create.c index 3a581f9d54..b1770dbd7f 100644 --- a/libpkg/pkg_create.c +++ b/libpkg/pkg_create.c @@ -200,7 +200,8 @@ pkg_create_archive(struct pkg *pkg, struct pkg_create *pc, unsigned required_fla } if (packing_init(&pkg_archive, pkg_path, pc->format, - pc->compression_level, pc->timestamp, pc->overwrite, false) != EPKG_OK) { + pc->compression_level, pc->compression_threads, pc->timestamp, + pc->overwrite, false) != EPKG_OK) { pkg_archive = NULL; } @@ -242,6 +243,7 @@ pkg_create_new(void) pc = xcalloc(1, sizeof(*pc)); pc->format = DEFAULT_COMPRESSION; pc->compression_level = ctx.compression_level; + pc->compression_threads = ctx.compression_threads; pc->timestamp = (time_t) -1; pc->overwrite = true; pc->expand_manifest = false; @@ -279,6 +281,12 @@ pkg_create_set_compression_level(struct pkg_create *pc, int clevel) pc->compression_level = clevel; } +void +pkg_create_set_compression_threads(struct pkg_create *pc, unsigned cthreads) +{ + pc->compression_threads = cthreads; +} + void pkg_create_set_expand_manifest(struct pkg_create *pc, bool expand) { diff --git a/libpkg/pkg_repo_create.c b/libpkg/pkg_repo_create.c index bff281df04..23d6d7a076 100644 --- a/libpkg/pkg_repo_create.c +++ b/libpkg/pkg_repo_create.c @@ -961,7 +961,7 @@ pkg_repo_pack_db(const char *name, const char *archive, char *path, sig = NULL; pub = NULL; - if (packing_init(&pack, archive, meta->packing_format, 0, (time_t)-1, true, true) != EPKG_OK) + if (packing_init(&pack, archive, meta->packing_format, 0, 1, (time_t)-1, true, true) != EPKG_OK) return (EPKG_FATAL); if (keyinfo != NULL) { diff --git a/libpkg/private/pkg.h b/libpkg/private/pkg.h index 252cd09ab2..df861ed599 100644 --- a/libpkg/private/pkg.h +++ b/libpkg/private/pkg.h @@ -155,6 +155,7 @@ struct pkg_ctx { const char *dbdir; const char *cachedir; int compression_level; + unsigned compression_threads; int rootfd; int cachedirfd; int dbdirfd; @@ -273,6 +274,7 @@ struct pkg_create { bool overwrite; bool expand_manifest; int compression_level; + unsigned compression_threads; pkg_formats format; time_t timestamp; const char *rootdir; @@ -680,7 +682,9 @@ int pkg_jobs_resolv(struct pkg_jobs *jobs); struct packing; -int packing_init(struct packing **pack, const char *path, pkg_formats format, int clevel, time_t timestamp, bool overwrite, bool archive_symlink); +int packing_init(struct packing **pack, const char *path, pkg_formats format, + int clevel, unsigned cthreads, time_t timestamp, bool overwrite, + bool archive_symlink); int packing_append_file_attr(struct packing *pack, const char *filepath, const char *newpath, const char *uname, const char *gname, mode_t perm, u_long fflags); diff --git a/src/create.c b/src/create.c index aada64f869..6169ab4d82 100644 --- a/src/create.c +++ b/src/create.c @@ -62,13 +62,13 @@ struct pkg_entry *pkg_head = NULL; void usage_create(void) { - fprintf(stderr, "Usage: pkg create [-eOhnqv] [-f format] [-l level] " + fprintf(stderr, "Usage: pkg create [-eOhnqv] [-T threads] [-f format] [-l level] " "[-o outdir] [-p plist] [-r rootdir] -m metadatadir\n"); - fprintf(stderr, "Usage: pkg create [-eOhnqv] [-f format] [-l level] " + fprintf(stderr, "Usage: pkg create [-eOhnqv] [-T threads] [-f format] [-l level] " "[-o outdir] [-r rootdir] -M manifest\n"); - fprintf(stderr, " pkg create [-eOhgnqvx] [-f format] [-l level] " + fprintf(stderr, " pkg create [-eOhgnqvx] [-T threads] [-f format] [-l level] " "[-o outdir] [-r rootdir] pkg-name ...\n"); - fprintf(stderr, " pkg create [-eOhnqv] [-f format] [-l level] " + fprintf(stderr, " pkg create [-eOhnqv] [-T threads] [-f format] [-l level] " "[-o outdir] [-r rootdir] -a\n\n"); fprintf(stderr, "For more information see 'pkg help create'.\n"); } @@ -179,6 +179,7 @@ exec_create(int argc, char **argv) int ch; int level; int ret; + unsigned threads = 1; bool hash = false; bool overwrite = true; bool expand_manifest = false; @@ -210,11 +211,12 @@ exec_create(int argc, char **argv) { "plist", required_argument, NULL, 'p' }, { "quiet", no_argument, NULL, 'q' }, { "timestamp", required_argument, NULL, 't' }, + { "threads", required_argument, NULL, 'T' }, { "verbose", no_argument, NULL, 'v' }, { NULL, 0, NULL, 0 }, }; - while ((ch = getopt_long(argc, argv, "+aeghxf:l:r:m:M:no:p:qvt:", longopts, NULL)) != -1) { + while ((ch = getopt_long(argc, argv, "+aeghxf:l:r:m:M:no:p:qvt:T:", longopts, NULL)) != -1) { switch (ch) { case 'a': match = MATCH_ALL; @@ -277,6 +279,14 @@ exec_create(int argc, char **argv) return (EXIT_FAILURE); } break; + case 'T': + endptr = NULL; + threads = (unsigned)strtoul(optarg, &endptr, 10); + if (*endptr != '\0') { + warnx("Invalid compression threads %s", optarg); + return (EXIT_FAILURE); + } + break; case 'v': quiet = false; break; @@ -315,6 +325,7 @@ exec_create(int argc, char **argv) warnx("unknown format %s, using the default", format); } pkg_create_set_compression_level(pc, level); + pkg_create_set_compression_threads(pc, threads); pkg_create_set_overwrite(pc, overwrite); pkg_create_set_rootdir(pc, rootdir); pkg_create_set_output_dir(pc, outdir);