From 581dda60f931c6a474a283f6904e496d9c7c6d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sun, 8 Oct 2023 11:33:44 -0700 Subject: [PATCH] feat(perf): be smarter about reflinks and mkdirp --- Cargo.lock | 269 ++++++++++-------- Cargo.toml | 19 +- crates/nassun/src/fetch/git.rs | 2 +- crates/nassun/src/package.rs | 89 +++--- crates/nassun/src/resolver.rs | 7 +- crates/nassun/src/tarball.rs | 126 ++++---- crates/node-maintainer/src/linkers/hoisted.rs | 46 +-- .../node-maintainer/src/linkers/isolated.rs | 81 +++--- crates/node-maintainer/src/linkers/mod.rs | 80 +++++- crates/node-maintainer/src/maintainer.rs | 7 +- crates/node-maintainer/src/resolver.rs | 11 +- src/apply_args.rs | 8 +- 12 files changed, 456 insertions(+), 289 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0c09e2a..b3746af2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -68,15 +68,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -145,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener 2.5.3", + "event-listener", "futures-core", ] @@ -177,9 +177,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f2db9467baa66a700abce2a18c5ad793f6f83310aca1284796fc3921d113fd" +checksum = "2c1da3ae8dabd9c00f453a329dfe1fb28da3c0a72e2478cdcd93171740c20499" dependencies = [ "async-lock", "async-task", @@ -219,7 +219,7 @@ dependencies = [ "log", "parking", "polling", - "rustix 0.37.23", + "rustix 0.37.24", "slab", "socket2 0.4.9", "waker-fn", @@ -231,42 +231,24 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener 2.5.3", + "event-listener", ] [[package]] name = "async-process" -version = "1.8.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf012553ce51eb7aa6dc2143804cc8252bd1cb681a1c5cb7fa94ca88682dee1d" +checksum = "7a9d28b1d97e08915212e2e45310d47854eafa69600756fc735fb788f75199c9" dependencies = [ "async-io", "async-lock", - "async-signal", + "autocfg", "blocking", "cfg-if", - "event-listener 3.0.0", + "event-listener", "futures-lite", - "rustix 0.38.14", - "windows-sys 0.48.0", -] - -[[package]] -name = "async-signal" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4af361a844928cb7d36590d406709473a1b574f443094422ef166daa3b493208" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "concurrent-queue", - "futures-core", - "futures-io", - "libc", - "signal-hook-registry", - "slab", + "rustix 0.37.24", + "signal-hook", "windows-sys 0.48.0", ] @@ -326,7 +308,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -477,9 +459,9 @@ dependencies = [ [[package]] name = "bytecount" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" [[package]] name = "bytemuck" @@ -489,9 +471,9 @@ checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -535,6 +517,32 @@ dependencies = [ "walkdir", ] +[[package]] +name = "cacache" +version = "12.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142316461ed3a3dfcba10417317472da5bfd0461e4d276bf7c07b330766d9490" +dependencies = [ + "async-std", + "digest", + "either", + "futures 0.3.28", + "hex", + "libc", + "memmap2", + "miette", + "reflink-copy", + "serde", + "serde_derive", + "serde_json", + "sha1", + "sha2", + "ssri", + "tempfile", + "thiserror", + "walkdir", +] + [[package]] name = "cc" version = "1.0.83" @@ -575,9 +583,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -585,9 +593,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -604,7 +612,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -956,9 +964,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", @@ -981,17 +989,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e56284f00d94c1bc7fd3c77027b4623c88c1f53d8d2394c6199f2921dea325" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -1196,7 +1193,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1329,9 +1326,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "heck" @@ -1402,7 +1399,7 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "cacache", + "cacache 11.7.1", "http", "http-cache-semantics", "httpdate", @@ -1592,12 +1589,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] @@ -1622,9 +1619,9 @@ checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" [[package]] name = "insta" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e02c584f4595792d09509a94cdb92a3cef7592b1eb2d9877ee6f527062d0ea" +checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686" dependencies = [ "console", "lazy_static", @@ -1700,7 +1697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.14", + "rustix 0.38.17", "windows-sys 0.48.0", ] @@ -1797,9 +1794,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linked-hash-map" @@ -1815,9 +1812,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" [[package]] name = "lock_api" @@ -1861,9 +1858,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" @@ -1903,7 +1900,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1979,7 +1976,7 @@ dependencies = [ "async-trait", "backon", "bincode", - "cacache", + "cacache 12.0.0", "console_error_panic_hook", "dashmap", "flate2", @@ -2179,7 +2176,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2467,7 +2464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.1", + "indexmap 2.0.2", ] [[package]] @@ -2493,7 +2490,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2612,9 +2609,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" dependencies = [ "unicode-ident", ] @@ -2762,9 +2759,9 @@ dependencies = [ [[package]] name = "reflink-copy" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e3947399fd46f412918bafde71ec68f9b3505f11ef082eeb80bc7fdf4d7caf" +checksum = "d7e3e017e993f86feeddf8a7fb609ca49f89082309e328e27aefd4a25bb317a4" dependencies = [ "cfg-if", "ioctl-sys", @@ -2773,13 +2770,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", + "regex-automata 0.3.9", "regex-syntax 0.7.5", ] @@ -2794,9 +2791,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick", "memchr", @@ -2826,9 +2823,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "async-compression 0.4.3", "base64 0.21.4", @@ -2853,6 +2850,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-util", @@ -2991,9 +2989,9 @@ dependencies = [ [[package]] name = "roxmltree" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f595a457b6b8c6cda66a48503e92ee8d19342f905948f29c383200ec9eb1d8" +checksum = "862340e351ce1b271a378ec53f304a5558f7db87f3769dc655a8f6ecbb68b302" dependencies = [ "xmlparser", ] @@ -3015,9 +3013,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "4279d76516df406a8bd37e7dff53fd37d1a093f997a3c34a5c21658c126db06d" dependencies = [ "bitflags 1.3.2", "errno", @@ -3029,14 +3027,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" dependencies = [ "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.7", + "linux-raw-sys 0.4.8", "windows-sys 0.48.0", ] @@ -3268,7 +3266,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3279,7 +3277,7 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3288,7 +3286,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -3352,9 +3350,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b21f559e07218024e7e9f90f96f601825397de0e25420135f7f952453fed0b" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -3365,6 +3363,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -3551,15 +3559,36 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -3595,7 +3624,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.1", "redox_syscall 0.3.5", - "rustix 0.38.14", + "rustix 0.38.17", "windows-sys 0.48.0", ] @@ -3647,7 +3676,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3659,7 +3688,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "test-case-core", ] @@ -3691,7 +3720,7 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3810,7 +3839,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3883,7 +3912,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3964,7 +3993,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -4062,9 +4091,9 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "ureq" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" +checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3" dependencies = [ "base64 0.21.4", "log", @@ -4268,7 +4297,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -4302,7 +4331,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4366,7 +4395,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.14", + "rustix 0.38.17", ] [[package]] @@ -4630,9 +4659,9 @@ dependencies = [ [[package]] name = "xmlparser" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" diff --git a/Cargo.toml b/Cargo.toml index b4e63085..9c2c4433 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ backon = "0.4.0" base64 = "0.21.2" bincode = "1.3.1" bytecount = "0.6.0" -cacache = "11.6.0" +cacache = "12.0.0" chrono = "0.4.23" chrono-humanize = "0.0.11" clap = "4.2.1" @@ -159,6 +159,10 @@ which = "4.0.2" wiremock = "0.5.17" reqwest-retry = "0.2.2" +# [patch.crates-io] +# cacache = { path = "../cacache-rs" } +# reflink-copy = { path = "../reflink-copy" } + # Config for 'cargo dist' [workspace.metadata.dist] # The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) @@ -168,7 +172,12 @@ rust-toolchain-version = "1.72.1" # CI backends to support ci = ["github"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"] +targets = [ + "x86_64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", +] # The installers to generate for each app installers = ["shell", "powershell", "npm", "msi"] # The archive format to use for windows builds (defaults .zip) @@ -190,8 +199,10 @@ path = "src/main.rs" [profile.release] lto = "thin" opt-level = 3 -strip = true -debug = false +# strip = true +# debug = false +strip = false +debug = true # The profile that 'cargo dist' will build with [profile.dist] diff --git a/crates/nassun/src/fetch/git.rs b/crates/nassun/src/fetch/git.rs index 62b931e6..c63c939e 100644 --- a/crates/nassun/src/fetch/git.rs +++ b/crates/nassun/src/fetch/git.rs @@ -93,7 +93,7 @@ impl GitFetcher { async fn fetch_tarball(&self, dir: &Path, tarball: &Url) -> Result<()> { let tarball = self.client.stream_external(tarball).await?; Tarball::new_unchecked(tarball) - .extract_from_tarball_data(dir, None, false) + .extract_from_tarball_data(dir, None, crate::ExtractMode::AutoHardlink) .await?; Ok(()) } diff --git a/crates/nassun/src/package.rs b/crates/nassun/src/package.rs index 4f07228b..bffe46d7 100644 --- a/crates/nassun/src/package.rs +++ b/crates/nassun/src/package.rs @@ -18,6 +18,32 @@ use crate::tarball::Tarball; #[cfg(not(target_arch = "wasm32"))] use crate::tarball::TarballIndex; +#[cfg(not(target_arch = "wasm32"))] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ExtractMode { + /// Automatically decide whether to Copy or Reflink, based on fallbacks. Will never hardlink. + #[default] + Auto, + /// Copy contents from the cache in their entirety. + Copy, + /// Reflink contents from the cache instead of doing full copies. + Reflink, + /// Try to hard link contents from the cache. Fall back to reflink, then copy if that fails. + AutoHardlink, + /// Hard link contents from the cache instead of doing full copies. + Hardlink, +} + +#[cfg(not(target_arch = "wasm32"))] +impl ExtractMode { + pub fn is_copy(&self) -> bool { + matches!( + self, + ExtractMode::Copy | ExtractMode::Auto | ExtractMode::Reflink + ) + } +} + /// A resolved package. A concrete version has been determined from its /// PackageSpec by the version resolver. #[derive(Clone)] @@ -131,13 +157,13 @@ impl Package { pub async fn extract_to_dir( &self, dir: impl AsRef, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { - async fn inner(me: &Package, dir: &Path, prefer_copy: bool) -> Result { - me.extract_to_dir_inner(dir, me.resolved.integrity(), prefer_copy) + async fn inner(me: &Package, dir: &Path, extract_mode: ExtractMode) -> Result { + me.extract_to_dir_inner(dir, me.resolved.integrity(), extract_mode) .await } - inner(self, dir.as_ref(), prefer_copy).await + inner(self, dir.as_ref(), extract_mode).await } /// Extract tarball to a directory, optionally caching its contents. The @@ -147,12 +173,12 @@ impl Package { pub async fn extract_to_dir_unchecked( &self, dir: impl AsRef, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { - async fn inner(me: &Package, dir: &Path, prefer_copy: bool) -> Result { - me.extract_to_dir_inner(dir, None, prefer_copy).await + async fn inner(me: &Package, dir: &Path, extract_mode: ExtractMode) -> Result { + me.extract_to_dir_inner(dir, None, extract_mode).await } - inner(self, dir.as_ref(), prefer_copy).await + inner(self, dir.as_ref(), extract_mode).await } /// Extract tarball to a directory, optionally caching its contents. The @@ -163,17 +189,17 @@ impl Package { &self, dir: impl AsRef, sri: Integrity, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { async fn inner( me: &Package, dir: &Path, sri: Integrity, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { - me.extract_to_dir_inner(dir, Some(&sri), prefer_copy).await + me.extract_to_dir_inner(dir, Some(&sri), extract_mode).await } - inner(self, dir.as_ref(), sri, prefer_copy).await + inner(self, dir.as_ref(), sri, extract_mode).await } #[cfg(not(target_arch = "wasm32"))] @@ -181,7 +207,7 @@ impl Package { &self, dir: &Path, integrity: Option<&Integrity>, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { if let Some(sri) = integrity { if let Some(cache) = self.cache.as_deref() { @@ -190,7 +216,7 @@ impl Package { { let sri = sri.clone(); match self - .extract_from_cache(dir, cache, entry, prefer_copy) + .extract_from_cache(dir, cache, entry, extract_mode) .await { Ok(_) => return Ok(sri), @@ -209,7 +235,7 @@ impl Package { return self .tarball_checked(sri) .await? - .extract_from_tarball_data(dir, self.cache.as_deref(), prefer_copy) + .extract_from_tarball_data(dir, self.cache.as_deref(), extract_mode) .await; } } @@ -217,18 +243,18 @@ impl Package { return self .tarball_checked(sri.clone()) .await? - .extract_from_tarball_data(dir, self.cache.as_deref(), prefer_copy) + .extract_from_tarball_data(dir, self.cache.as_deref(), extract_mode) .await; } } self.tarball_checked(sri.clone()) .await? - .extract_from_tarball_data(dir, self.cache.as_deref(), prefer_copy) + .extract_from_tarball_data(dir, self.cache.as_deref(), extract_mode) .await } else { self.tarball_unchecked() .await? - .extract_from_tarball_data(dir, self.cache.as_deref(), prefer_copy) + .extract_from_tarball_data(dir, self.cache.as_deref(), extract_mode) .await } } @@ -239,13 +265,13 @@ impl Package { dir: &Path, cache: &Path, entry: cacache::Metadata, - mut prefer_copy: bool, + mut extract_mode: ExtractMode, ) -> Result<()> { let dir = PathBuf::from(dir); let cache = PathBuf::from(cache); let name = self.name().to_owned(); async_std::task::spawn_blocking(move || { - let mut created = std::collections::HashSet::new(); + let created = dashmap::DashSet::new(); let index = rkyv::check_archived_root::( entry .raw_metadata @@ -253,22 +279,19 @@ impl Package { .ok_or_else(|| NassunError::CacheMissingIndexError(name))?, ) .map_err(|e| NassunError::DeserializeCacheError(e.to_string()))?; - prefer_copy = index.should_copy || prefer_copy; + extract_mode = if index.should_copy && !extract_mode.is_copy() { + // In general, if reflinks are supported, we would have + // received them as extract_mode already. So there's no need + // to try and do a fallback here. + ExtractMode::Copy + } else { + extract_mode + }; for (archived_path, (sri, mode)) in index.files.iter() { let sri: Integrity = sri.parse()?; let path = dir.join(&archived_path[..]); let parent = PathBuf::from(path.parent().expect("this will always have a parent")); - if !created.contains(&parent) { - std::fs::create_dir_all(path.parent().expect("this will always have a parent")) - .map_err(|e| { - NassunError::ExtractIoError( - e, - Some(PathBuf::from(path.parent().unwrap())), - "creating destination directory for tarball.".into(), - ) - })?; - created.insert(parent); - } + crate::tarball::mkdirp(&parent, &created)?; let mode = if index.bin_paths.contains(archived_path) { *mode | 0o111 @@ -276,7 +299,7 @@ impl Package { *mode }; - crate::tarball::extract_from_cache(&cache, &sri, &path, prefer_copy, mode)?; + crate::tarball::extract_from_cache(&cache, &sri, &path, extract_mode, mode)?; } Ok::<_, NassunError>(()) }) diff --git a/crates/nassun/src/resolver.rs b/crates/nassun/src/resolver.rs index 9661275e..87ec99fb 100644 --- a/crates/nassun/src/resolver.rs +++ b/crates/nassun/src/resolver.rs @@ -54,12 +54,7 @@ impl std::fmt::Debug for PackageResolution { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use PackageResolution::*; match self { - Npm { - tarball, - name, - version, - .. - } => write!(f, "{name}@{version} ({tarball})"), + Npm { name, version, .. } => write!(f, "{name}@{version}"), Dir { path, name } => write!(f, "{name}@{}", path.to_string_lossy()), Git { name, info } => write!(f, "{name}@{info}"), } diff --git a/crates/nassun/src/tarball.rs b/crates/nassun/src/tarball.rs index 8d1780a9..88f64bef 100644 --- a/crates/nassun/src/tarball.rs +++ b/crates/nassun/src/tarball.rs @@ -1,6 +1,5 @@ #[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; -use std::collections::HashSet; #[cfg(not(target_arch = "wasm32"))] use std::io::Write; #[cfg(not(target_arch = "wasm32"))] @@ -34,6 +33,8 @@ use crate::entries::{Entries, Entry}; #[cfg(not(target_arch = "wasm32"))] use crate::error::IoContext; use crate::error::{NassunError, Result}; +#[cfg(not(target_arch = "wasm32"))] +use crate::package::ExtractMode; use crate::TarballStream; #[cfg(not(target_arch = "wasm32"))] @@ -74,14 +75,14 @@ impl Tarball { mut self, dir: &Path, cache: Option<&Path>, - prefer_copy: bool, + extract_mode: ExtractMode, ) -> Result { let integrity = self.integrity.take(); let temp = self.into_temp().await?; let dir = PathBuf::from(dir); let cache = cache.map(PathBuf::from); async_std::task::spawn_blocking(move || { - temp.extract_to_dir(&dir, integrity, cache.as_deref(), prefer_copy) + temp.extract_to_dir(&dir, integrity, cache.as_deref(), extract_mode) }) .await } @@ -207,11 +208,12 @@ impl TempTarball { dir: &Path, tarball_integrity: Option, cache: Option<&Path>, - mut prefer_copy: bool, + mut extract_mode: ExtractMode, ) -> Result { let mut build_mani: Option = None; let mut tarball_index = TarballIndex::default(); let mut drain_buf = [0u8; 1024 * 8]; + let created = dashmap::DashSet::new(); self.rewind().io_context(|| { format!( @@ -229,13 +231,7 @@ impl TempTarball { NassunError::ExtractIoError(e, None, "getting tarball entries iterator".into()) })?; - std::fs::create_dir_all(dir).map_err(|e| { - NassunError::ExtractIoError( - e, - Some(PathBuf::from(dir)), - "creating destination directory for tarball.".into(), - ) - })?; + mkdirp(dir, &created)?; for file in files { let mut file = file.map_err(|e| { @@ -254,21 +250,9 @@ impl TempTarball { .unwrap_or_else(|| entry_path.as_ref()) .to_path_buf(); let path = dir.join(&entry_subpath); - let mut created = HashSet::new(); if let tar::EntryType::Regular = header.entry_type() { let parent = path.parent().unwrap(); - if !created.contains(parent) { - std::fs::create_dir_all(parent).map_err(|e| { - NassunError::ExtractIoError( - e, - Some(path.parent().unwrap().into()), - "creating parent directory for entry.".into(), - ) - })?; - for path in parent.ancestors() { - created.insert(path.to_path_buf()); - } - } + mkdirp(parent, &created)?; if let Some(cache) = cache { let mut writer = WriteOpts::new() @@ -288,7 +272,7 @@ impl TempTarball { .commit() .map_err(|e| NassunError::ExtractCacheError(e, Some(path.clone())))?; - extract_from_cache(cache, &sri, &path, prefer_copy, mode)?; + extract_from_cache(cache, &sri, &path, extract_mode, mode)?; let entry_subpath = entry_subpath.to_string_lossy().to_string(); @@ -305,16 +289,15 @@ impl TempTarball { if ["preinstall", "install", "postinstall"] .iter() .any(|s| manifest.scripts.contains_key(*s)) - || !manifest.bin.is_empty() { tarball_index.should_copy = true; - if !prefer_copy { - prefer_copy = true; + if !extract_mode.is_copy() { + extract_mode = ExtractMode::Auto; for (entry, (sri, mode)) in &tarball_index.files { let path = dir.join(entry); std::fs::remove_file(&path).io_context(|| format!("Failed to remove target file while extracting a new version, at {}.", path.display()))?; let sri = sri.parse()?; - extract_from_cache(cache, &sri, &path, prefer_copy, *mode)?; + extract_from_cache(cache, &sri, &path, extract_mode, *mode)?; } } } @@ -451,27 +434,34 @@ pub(crate) fn extract_from_cache( cache: &Path, sri: &Integrity, to: &Path, - prefer_copy: bool, + extract_mode: ExtractMode, #[allow(unused_variables)] mode: u32, ) -> Result<()> { - if prefer_copy { - copy_from_cache(cache, sri, to)?; - } else { - // HACK: This is horrible, but on wsl2 (at least), this - // was sometimes crashing with an ENOENT (?!), which - // really REALLY shouldn't happen. So we just retry a few - // times and hope the problem goes away. - let op = || hard_link_from_cache(cache, sri, to); - op.retry(&ConstantBuilder::default().with_delay(Duration::from_millis(50))) - .notify(|err, wait| { - tracing::debug!( - "Error hard linking from cache: {}. Retrying after {}ms", - err, - wait.as_micros() / 1000 - ) - }) - .call() - .or_else(|_| copy_from_cache(cache, sri, to))?; + match extract_mode { + ExtractMode::Auto => { + reflink_from_cache(cache, sri, to).or_else(|_| copy_from_cache(cache, sri, to))?; + } + ExtractMode::AutoHardlink | ExtractMode::Hardlink => { + // HACK: This is horrible, but on wsl2 (at least), this + // was sometimes crashing with an ENOENT (?!), which + // really REALLY shouldn't happen. So we just retry a few + // times and hope the problem goes away. + (|| hard_link_from_cache(cache, sri, to)) + .retry(&ConstantBuilder::default().with_delay(Duration::from_millis(50))) + .notify(|err, wait| { + tracing::debug!( + "Error hard linking from cache: {}. Retrying after {}ms", + err, + wait.as_micros() / 1000 + ) + }) + .call() + // NOTE: we still want the operation to complete if hard linking fails. + .or_else(|_| reflink_from_cache(cache, sri, to)) + .or_else(|_| copy_from_cache(cache, sri, to))?; + } + ExtractMode::Copy => copy_from_cache(cache, sri, to)?, + ExtractMode::Reflink => reflink_from_cache(cache, sri, to)?, } #[cfg(unix)] { @@ -517,9 +507,49 @@ fn copy_from_cache(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { Ok(()) } +#[cfg(not(target_arch = "wasm32"))] +fn reflink_from_cache(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { + cacache::reflink_hash_unchecked_sync(cache, sri, to) + .map_err(|e| NassunError::ExtractCacheError(e, Some(PathBuf::from(to))))?; + Ok(()) +} + #[cfg(not(target_arch = "wasm32"))] fn hard_link_from_cache(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { cacache::hard_link_hash_unchecked_sync(cache, sri, to) .map_err(|e| NassunError::ExtractCacheError(e, Some(PathBuf::from(to))))?; Ok(()) } + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn mkdirp(path: &Path, cache: &dashmap::DashSet) -> Result<()> { + if !cache.contains(path) { + let grandpa_present = if let Some(grandpa) = path.parent() { + cache.contains(grandpa) + } else { + true + }; + if grandpa_present { + std::fs::create_dir(path).map_err(|e| { + NassunError::ExtractIoError( + e, + Some(path.parent().unwrap().into()), + "creating parent directory for entry.".into(), + ) + })?; + cache.insert(path.to_path_buf()); + } else { + std::fs::create_dir_all(path).map_err(|e| { + NassunError::ExtractIoError( + e, + Some(path.parent().unwrap().into()), + "creating parent directory for entry.".into(), + ) + })?; + for path in path.ancestors() { + cache.insert(path.to_path_buf()); + } + } + } + Ok(()) +} diff --git a/crates/node-maintainer/src/linkers/hoisted.rs b/crates/node-maintainer/src/linkers/hoisted.rs index bb9f137a..619c21b9 100644 --- a/crates/node-maintainer/src/linkers/hoisted.rs +++ b/crates/node-maintainer/src/linkers/hoisted.rs @@ -4,8 +4,10 @@ use std::path::PathBuf; use std::sync::atomic::AtomicUsize; use std::sync::{atomic, Arc}; +use dashmap::DashSet; use futures::lock::Mutex; use futures::{StreamExt, TryStreamExt}; +use nassun::ExtractMode; use oro_common::BuildManifest; use petgraph::stable_graph::NodeIndex; use unicase::UniCase; @@ -19,6 +21,7 @@ use super::LinkerOptions; pub(crate) struct HoistedLinker { pub(crate) pending_rebuild: Arc>>, + pub(crate) mkdir_cache: Arc>, pub(crate) opts: LinkerOptions, } @@ -26,6 +29,7 @@ impl HoistedLinker { pub fn new(opts: LinkerOptions) -> Self { Self { pending_rebuild: Arc::new(Mutex::new(HashSet::new())), + mkdir_cache: Arc::new(DashSet::new()), opts, } } @@ -247,17 +251,20 @@ impl HoistedLinker { let total = graph.inner.node_count(); let total_completed = Arc::new(AtomicUsize::new(0)); let node_modules = root.join("node_modules"); - std::fs::create_dir_all(&node_modules).io_context(|| { - format!( - "Failed to create project node_modules directory at {}.", - node_modules.display() - ) - })?; - let prefer_copy = self.opts.prefer_copy - || match self.opts.cache.as_deref() { - Some(cache) => super::supports_reflink(cache, &node_modules), - None => false, - }; + super::mkdirp(&node_modules, &self.mkdir_cache)?; + let extract_mode = if let Some(cache) = self.opts.cache.as_deref() { + if super::supports_reflink(cache, &node_modules) { + ExtractMode::Reflink + } else if self.opts.prefer_copy { + ExtractMode::Copy + } else if super::supports_hardlink(cache, &node_modules) { + ExtractMode::Hardlink + } else { + ExtractMode::Copy + } + } else { + ExtractMode::AutoHardlink + }; stream .map(|idx| { Ok(( @@ -295,7 +302,7 @@ impl HoistedLinker { if !target_dir.exists() { graph[child_idx] .package - .extract_to_dir(&target_dir, prefer_copy) + .extract_to_dir(&target_dir, extract_mode) .await?; actually_extracted.fetch_add(1, atomic::Ordering::SeqCst); let target_dir = target_dir.clone(); @@ -318,8 +325,10 @@ impl HoistedLinker { } } + let elapsed = start.elapsed(); + if let Some(on_extract) = &self.opts.on_extract_progress { - on_extract(&graph[child_idx].package); + on_extract(&graph[child_idx].package, elapsed); } tracing::trace!( @@ -327,7 +336,7 @@ impl HoistedLinker { "Extracted {} to {} in {:?}ms. {}/{total} done.", graph[child_idx].package.name(), target_dir.display(), - start.elapsed().as_millis(), + elapsed.as_micros() / 1000, total_completed.fetch_add(1, atomic::Ordering::SeqCst) + 1, ); Ok::<_, NodeMaintainerError>(()) @@ -399,17 +408,12 @@ impl HoistedLinker { let to = target_dir.join(name); let from = package_dir.join(path); let name = name.clone(); + let mkdir_cache = self.mkdir_cache.clone(); async_std::task::spawn_blocking(move || { // We only create a symlink if the target bin exists. let target_dir = &target_dir; if from.symlink_metadata().is_ok() { - std::fs::create_dir_all(target_dir).io_context(|| { - format!( - "Failed to create .bin dir at {} while linking {} bin.", - target_dir.display(), - name, - ) - })?; + super::mkdirp(target_dir, &mkdir_cache)?; // TODO: use a DashMap here to prevent race conditions, maybe? if let Ok(meta) = to.symlink_metadata() { if meta.is_dir() { diff --git a/crates/node-maintainer/src/linkers/isolated.rs b/crates/node-maintainer/src/linkers/isolated.rs index 7d02aba5..c184769f 100644 --- a/crates/node-maintainer/src/linkers/isolated.rs +++ b/crates/node-maintainer/src/linkers/isolated.rs @@ -9,6 +9,7 @@ use std::{ use dashmap::DashSet; use futures::{lock::Mutex, StreamExt, TryStreamExt}; +use nassun::ExtractMode; use oro_common::BuildManifest; use petgraph::{stable_graph::NodeIndex, visit::EdgeRef, Direction}; use ssri::Integrity; @@ -20,7 +21,7 @@ use super::LinkerOptions; pub(crate) struct IsolatedLinker { pub(crate) pending_rebuild: Arc>>, pub(crate) pending_bin_link: Arc>>, - pub(crate) created_dirs: Arc>, + pub(crate) mkdir_cache: Arc>, pub(crate) opts: LinkerOptions, } @@ -29,7 +30,7 @@ impl IsolatedLinker { Self { pending_rebuild: Arc::new(Mutex::new(HashSet::new())), pending_bin_link: Arc::new(Mutex::new(BinaryHeap::new())), - created_dirs: Arc::new(DashSet::new()), + mkdir_cache: Arc::new(DashSet::new()), opts, } } @@ -349,17 +350,20 @@ impl IsolatedLinker { let total = graph.inner.node_count(); let total_completed = Arc::new(AtomicUsize::new(0)); let node_modules = root.join("node_modules"); - std::fs::create_dir_all(&node_modules).io_context(|| { - format!( - "Failed to create node_modules directory at {} for extraction.", - node_modules.display() - ) - })?; - let prefer_copy = self.opts.prefer_copy - || match self.opts.cache.as_deref() { - Some(cache) => super::supports_reflink(cache, &node_modules), - None => false, - }; + super::mkdirp(&node_modules, &self.mkdir_cache)?; + let extract_mode = if let Some(cache) = self.opts.cache.as_deref() { + if super::supports_reflink(cache, &node_modules) { + ExtractMode::Reflink + } else if self.opts.prefer_copy { + ExtractMode::Copy + } else if super::supports_hardlink(cache, &node_modules) { + ExtractMode::Hardlink + } else { + ExtractMode::Copy + } + } else { + ExtractMode::AutoHardlink + }; stream .map(|idx| { Ok(( @@ -403,7 +407,7 @@ impl IsolatedLinker { if !target_dir.exists() { graph[child_idx] .package - .extract_to_dir(&target_dir, prefer_copy) + .extract_to_dir(&target_dir, extract_mode) .await?; actually_extracted.fetch_add(1, atomic::Ordering::SeqCst); let target_dir = target_dir.clone(); @@ -436,8 +440,10 @@ impl IsolatedLinker { ) .await?; + let elapsed = start.elapsed(); + if let Some(on_extract) = &self.opts.on_extract_progress { - on_extract(&graph[child_idx].package); + on_extract(&graph[child_idx].package, elapsed); } tracing::trace!( @@ -445,7 +451,7 @@ impl IsolatedLinker { "Extracted {} to {} in {:?}ms. {}/{total} done.", graph[child_idx].package.name(), target_dir.display(), - start.elapsed().as_millis(), + elapsed.as_micros() / 1000, total_completed.fetch_add(1, atomic::Ordering::SeqCst) + 1, ); @@ -520,26 +526,30 @@ impl IsolatedLinker { dep_nm_entry.parent().expect("must have a parent"), ) .expect("this should never fail"); - let created_dirs = self.created_dirs.clone(); + let mkdir_cache = self.mkdir_cache.clone(); async_std::task::spawn_blocking(move || { let path = dep_nm_entry.parent().expect("definitely has a parent"); - if !created_dirs.contains(path) { - std::fs::create_dir_all(path).io_context(|| { - format!("Failed to create directory for dependency in package store at {} while linking dep.", path.display()) - })?; - for path in path.ancestors() { - created_dirs.insert(path.to_path_buf()); - } - } + super::mkdirp(path, &mkdir_cache)?; if dep_nm_entry.symlink_metadata().is_err() { // We don't check the link target here because we assume prune() has already been run and removed any incorrect links. #[cfg(windows)] std::os::windows::fs::symlink_dir(&relative, &dep_nm_entry) - .or_else(|_| junction::create(&dep_store_dir, &dep_nm_entry)).map_err(|e| { - NodeMaintainerError::JunctionsNotSupported(dep_store_dir, dep_nm_entry, e) + .or_else(|_| junction::create(&dep_store_dir, &dep_nm_entry)) + .map_err(|e| { + NodeMaintainerError::JunctionsNotSupported( + dep_store_dir, + dep_nm_entry, + e, + ) })?; #[cfg(unix)] - std::os::unix::fs::symlink(&relative, &dep_nm_entry).io_context(|| format!("Failed to create symlink while linking dependency, from {} to {}.", relative.display(), dep_nm_entry.display()))?; + std::os::unix::fs::symlink(&relative, &dep_nm_entry).io_context(|| { + format!( + "Failed to create symlink while linking dependency, from {} to {}.", + relative.display(), + dep_nm_entry.display() + ) + })?; } Ok::<(), NodeMaintainerError>(()) }) @@ -581,23 +591,12 @@ impl IsolatedLinker { let to = dep_bin_dir.join(name); let from = dep_store_dir.join("node_modules").join(name).join(path); let name = name.clone(); - let created_dirs = self.created_dirs.clone(); + let mkdir_cache = self.mkdir_cache.clone(); async_std::task::spawn_blocking(move || { // We only create a symlink if the target bin exists. if from.symlink_metadata().is_ok() { let parent = to.parent().expect("has a parent"); - if !created_dirs.contains(parent) { - std::fs::create_dir_all(to.parent().expect("has a parent")) - .io_context(|| { - format!( - "Failed to create target bin directory at {}.", - to.parent().unwrap().display() - ) - })?; - for path in parent.ancestors() { - created_dirs.insert(path.to_path_buf()); - } - } + super::mkdirp(parent, &mkdir_cache)?; if let Ok(meta) = to.symlink_metadata() { if meta.is_dir() { std::fs::remove_dir_all(&to).io_context(|| { diff --git a/crates/node-maintainer/src/linkers/mod.rs b/crates/node-maintainer/src/linkers/mod.rs index d2f2b4ce..e6b52b4d 100644 --- a/crates/node-maintainer/src/linkers/mod.rs +++ b/crates/node-maintainer/src/linkers/mod.rs @@ -388,20 +388,58 @@ pub(crate) fn supports_reflink(src_dir: &Path, dest_dir: &Path) -> bool { let supports_reflink = reflink_copy::reflink(temp.path(), tempdir.path().join("b")) .map(|_| true) .map_err(|e| { - tracing::debug!( - "reflink support check failed. Files will be hard linked or copied. ({e})" - ); + tracing::debug!("reflink support check failed.({e})"); e }) .unwrap_or(false); if supports_reflink { - tracing::debug!("Verified reflink support. Extracted data will use copy-on-write reflinks instead of hard links or full copies.") + tracing::debug!("Verified reflink support") } supports_reflink } +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn supports_hardlink(src_dir: &Path, dest_dir: &Path) -> bool { + let temp = match tempfile::NamedTempFile::new_in(src_dir) { + Ok(t) => t, + Err(e) => { + tracing::debug!("error creating tempfile while checking for hardlink support: {e}."); + return false; + } + }; + match std::fs::write(&temp, "a") { + Ok(_) => {} + Err(e) => { + tracing::debug!("error writing to tempfile while checking for hardlink support: {e}."); + return false; + } + }; + let tempdir = match tempfile::TempDir::new_in(dest_dir) { + Ok(t) => t, + Err(e) => { + tracing::debug!( + "error creating destination tempdir while checking for hardlink support: {e}." + ); + return false; + } + }; + let supports_hardlink = std::fs::hard_link(temp.path(), tempdir.path().join("b")) + .map(|_| true) + .map_err(|e| { + tracing::debug!("hardlink support check failed. ({e})"); + e + }) + .unwrap_or(false); + + if supports_hardlink { + tracing::debug!("Verified hardlink support.") + } + + supports_hardlink +} + #[cfg(not(target_arch = "wasm32"))] pub(crate) fn link_bin(from: &Path, to: &Path) -> Result<(), NodeMaintainerError> { #[cfg(windows)] @@ -440,3 +478,37 @@ pub(crate) fn link_bin(from: &Path, to: &Path) -> Result<(), NodeMaintainerError } Ok(()) } + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn mkdirp( + path: &Path, + cache: &dashmap::DashSet, +) -> Result<(), NodeMaintainerError> { + if !cache.contains(path) { + let grandpa_present = if let Some(grandpa) = path.parent() { + cache.contains(grandpa) + } else { + true + }; + if grandpa_present { + std::fs::create_dir(path).io_context(|| { + format!( + "Failed to create directory {} while extracting.", + path.display() + ) + })?; + cache.insert(path.to_path_buf()); + } else { + std::fs::create_dir_all(path).io_context(|| { + format!( + "Failed to create directory {} while extracting.", + path.display() + ) + })?; + for path in path.ancestors() { + cache.insert(path.to_path_buf()); + } + } + } + Ok(()) +} diff --git a/crates/node-maintainer/src/maintainer.rs b/crates/node-maintainer/src/maintainer.rs index 31b751c7..ad7f9361 100644 --- a/crates/node-maintainer/src/maintainer.rs +++ b/crates/node-maintainer/src/maintainer.rs @@ -1,5 +1,6 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; #[cfg(not(target_arch = "wasm32"))] use async_std::fs; @@ -28,7 +29,7 @@ pub const META_FILE_NAME: &str = ".orogene-meta.kdl"; pub const STORE_DIR_NAME: &str = ".oro-store"; pub type ProgressAdded = Arc; -pub type ProgressHandler = Arc; +pub type ProgressHandler = Arc; pub type PruneProgress = Arc; pub type ScriptStartHandler = Arc; pub type ScriptLineHandler = Arc; @@ -239,7 +240,7 @@ impl NodeMaintainerOptions { pub fn on_resolve_progress(mut self, f: F) -> Self where - F: Fn(&Package) + Send + Sync + 'static, + F: Fn(&Package, Duration) + Send + Sync + 'static, { self.on_resolve_progress = Some(Arc::new(f)); self @@ -257,7 +258,7 @@ impl NodeMaintainerOptions { #[cfg(not(target_arch = "wasm32"))] pub fn on_extract_progress(mut self, f: F) -> Self where - F: Fn(&Package) + Send + Sync + 'static, + F: Fn(&Package, Duration) + Send + Sync + 'static, { self.on_extract_progress = Some(Arc::new(f)); self diff --git a/crates/node-maintainer/src/resolver.rs b/crates/node-maintainer/src/resolver.rs index 730b73b2..49cad175 100644 --- a/crates/node-maintainer/src/resolver.rs +++ b/crates/node-maintainer/src/resolver.rs @@ -2,6 +2,7 @@ use std::cmp::Ordering; use std::collections::{HashSet, VecDeque}; use std::path::Path; use std::sync::Arc; +use std::time::Instant; use async_std::sync::Mutex; #[cfg(not(target_arch = "wasm32"))] @@ -32,6 +33,7 @@ struct NodeDependency { spec: PackageSpec, dep_type: DepType, node_idx: NodeIndex, + start: Instant, } pub(crate) struct Resolver<'a> { @@ -118,6 +120,7 @@ impl<'a> Resolver<'a> { spec, dep_type, node_idx, + start: Instant::now(), }; if let Some(handler) = &self.on_resolution_added { @@ -126,7 +129,7 @@ impl<'a> Resolver<'a> { if let Some(_child_idx) = Self::satisfy_dependency(&mut self.graph, &dep)? { if let Some(handler) = &self.on_resolve_progress { - handler(&self.graph[_child_idx].package); + handler(&self.graph[_child_idx].package, dep.start.elapsed()); } } // Walk up the current hierarchy to see if we find a @@ -164,7 +167,7 @@ impl<'a> Resolver<'a> { q.push_back(child_idx); if let Some(handler) = &self.on_resolve_progress { - handler(&self.graph[child_idx].package); + handler(&self.graph[child_idx].package, dep.start.elapsed()); } continue; } @@ -220,7 +223,7 @@ impl<'a> Resolver<'a> { Self::satisfy_dependency(&mut self.graph, &dep)? { if let Some(handler) = &self.on_resolve_progress { - handler(&self.graph[_child_idx].package); + handler(&self.graph[_child_idx].package, dep.start.elapsed()); } continue; } @@ -236,7 +239,7 @@ impl<'a> Resolver<'a> { q.push_back(child_idx); if let Some(handler) = &self.on_resolve_progress { - handler(&self.graph[child_idx].package); + handler(&self.graph[child_idx].package, dep.start.elapsed()); } } } diff --git a/src/apply_args.rs b/src/apply_args.rs index 25bfb224..765971e3 100644 --- a/src/apply_args.rs +++ b/src/apply_args.rs @@ -182,20 +182,20 @@ impl ApplyArgs { .on_resolution_added(move || { Span::current().pb_inc_length(1); }) - .on_resolve_progress(move |pkg| { + .on_resolve_progress(move |pkg, elapsed| { let span = Span::current(); span.pb_inc(1); - span.pb_set_message(&format!("{:?}", pkg.resolved())); + span.pb_set_message(&format!("{:?} ({}ms)", pkg.resolved(), elapsed.as_micros() / 1000)); }) .on_prune_progress(move |path| { let span = Span::current(); span.pb_inc(1); span.pb_set_message(&format!("{}", path.display())); }) - .on_extract_progress(move |pkg| { + .on_extract_progress(move |pkg, elapsed| { let span = Span::current(); span.pb_inc(1); - span.pb_set_message(&format!("{:?}", pkg.resolved())) + span.pb_set_message(&format!("{:?} ({}ms)", pkg.resolved(), elapsed.as_micros() / 1000)) }) .on_script_start(|pkg, event| { let span = Span::current();