Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: [S

### Added

- `bashdep::install` downloads dependencies in parallel (up to 4 at a
time by default). Set `BASHDEP_JOBS` to change the concurrency;
`BASHDEP_JOBS=1` restores sequential downloads. Dry-run stays
sequential for stable preview output.

### Changed

### Fixed
Expand Down
70 changes: 65 additions & 5 deletions bashdep
Original file line number Diff line number Diff line change
Expand Up @@ -260,24 +260,84 @@ function bashdep::install() {
local dependencies=("$@")
local failures=0
local dep
local jobs="${BASHDEP_JOBS:-4}"

# Defer lockfile writes: each download records its entry in
# BASHDEP_LOCK_PENDING and we rewrite each destination lockfile once at
# the end, instead of re-sorting the whole file on every dependency.
BASHDEP_LOCK_DEFER=true
BASHDEP_LOCK_PENDING=()
for dep in "${dependencies[@]}"; do
bashdep::_classify_dep "$dep"
bashdep::setup_directory "$BASHDEP_DEP_DIR" || { failures=$((failures + 1)); continue; }
bashdep::download_url "$BASHDEP_DEP_URL" "$BASHDEP_DEP_DIR" || failures=$((failures + 1))
done

# Download concurrently when more than one dependency is requested and
# parallelism is enabled. Dry-run stays sequential so its preview output
# keeps a stable order.
if [[ "$jobs" -gt 1 && ${#dependencies[@]} -gt 1 ]] && ! bashdep::is_dry_run; then
bashdep::_install_parallel "$jobs" "${dependencies[@]}"
failures=$?
else
for dep in "${dependencies[@]}"; do
bashdep::_classify_dep "$dep"
bashdep::setup_directory "$BASHDEP_DEP_DIR" || { failures=$((failures + 1)); continue; }
bashdep::download_url "$BASHDEP_DEP_URL" "$BASHDEP_DEP_DIR" || failures=$((failures + 1))
done
fi

BASHDEP_LOCK_DEFER=false
bashdep::_lock_flush || failures=$((failures + 1))
BASHDEP_LOCK_PENDING=()

return $(( failures > 255 ? 255 : failures ))
}

# Internal: download all dependencies concurrently, at most $1 at a time.
# Each dependency runs in a background subshell that writes its lockfile
# entry to a per-job result file (subshell array mutations don't reach the
# parent); the parent then aggregates them into BASHDEP_LOCK_PENDING in
# dependency order. Returns the number of failed downloads.
function bashdep::_install_parallel() {
local jobs=$1; shift
local deps=("$@")
local failures=0 idx=0 running=0 p rf line
local results_dir; results_dir=$(mktemp -d) || return 1
local pids=()

local dep result_file
for dep in "${deps[@]}"; do
printf -v result_file '%s/%05d' "$results_dir" "$idx"
bashdep::_install_one_async "$dep" "$result_file" &
pids+=("$!")
idx=$((idx + 1)); running=$((running + 1))
if [[ $running -ge $jobs ]]; then
for p in "${pids[@]}"; do wait "$p" || failures=$((failures + 1)); done
pids=(); running=0
fi
done
for p in ${pids[@]+"${pids[@]}"}; do wait "$p" || failures=$((failures + 1)); done

# Aggregate in filename (= dependency) order so last-write-wins is stable.
for rf in "$results_dir"/*; do
[[ -s "$rf" ]] || continue
while IFS= read -r line; do BASHDEP_LOCK_PENDING+=("$line"); done < "$rf"
done
rm -rf "$results_dir"
# Cap before returning: `return` is 8-bit, so an uncapped count > 255
# would wrap (e.g. 300 -> 44). install re-caps regardless.
return $(( failures > 255 ? 255 : failures ))
}

# Internal: download one dependency (in a background subshell) and append
# its deferred lockfile entry to $2. Returns non-zero on failure.
function bashdep::_install_one_async() {
local dep=$1 result_file=$2 entry
bashdep::_classify_dep "$dep"
bashdep::setup_directory "$BASHDEP_DEP_DIR" || return 1
BASHDEP_LOCK_PENDING=()
bashdep::download_url "$BASHDEP_DEP_URL" "$BASHDEP_DEP_DIR" || return 1
for entry in ${BASHDEP_LOCK_PENDING[@]+"${BASHDEP_LOCK_PENDING[@]}"}; do
printf '%s\n' "$entry" >> "$result_file"
done
}

# Internal: classify a dependency string into URL and target directory.
# A dep ending in $BASHDEP_DEV_SUFFIX routes to $BASHDEP_DEV_DIR; otherwise $BASHDEP_DIR.
# Sets BASHDEP_DEP_URL and BASHDEP_DEP_DIR. Pure (no I/O).
Expand Down
4 changes: 3 additions & 1 deletion docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ DEPENDENCIES=(
bashdep::install "${DEPENDENCIES[@]}"
```

Returns the number of failed downloads (0 on success, capped at 255).
Downloads run in parallel, up to `BASHDEP_JOBS` at a time (default `4`).
Set `BASHDEP_JOBS=1` for sequential downloads. Returns the number of
failed downloads (0 on success, capped at 255).

## `bashdep::install_from`

Expand Down
7 changes: 5 additions & 2 deletions docs/behavior.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,11 @@ both by default.

## Error handling

- `bashdep::install` continues past failed downloads and returns the
failure count (capped at 255).
- `bashdep::install` downloads in parallel (up to `BASHDEP_JOBS`, default
`4`; set `BASHDEP_JOBS=1` for sequential). It continues past failed
downloads and returns the failure count (capped at 255). Lockfile
writes are batched into a single rewrite per directory after all
downloads finish, so concurrency never corrupts the lockfile.
- `bashdep::install_from` returns `1` if the file (default: `.bashdep`)
is missing or unreadable.
- `bashdep::setup` returns `1` on unknown params or non-boolean values
Expand Down
35 changes: 35 additions & 0 deletions tests/unit/bashdep_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,41 @@ function test_bashdep_install_batch_noop_when_all_skipped() {
assert_contains "tool" "$(cat "$TEST_DIR/.bashdep.lock")"
}

function test_bashdep_install_parallel_installs_all_deps() {
# shellcheck disable=SC2016
mock curl 'touch "$4"'
BASHDEP_DIR="$TEST_DIR"
BASHDEP_JOBS=4
bashdep::install \
"https://example.com/aaa" \
"https://example.com/bbb" \
"https://example.com/ccc" >/dev/null
assert_file_exists "$TEST_DIR/aaa"
assert_file_exists "$TEST_DIR/ccc"
local lock; lock=$(cat "$TEST_DIR/.bashdep.lock")
assert_contains "aaa" "$lock"
assert_contains "ccc" "$lock"
}

function test_bashdep_install_parallel_counts_failures() {
mock bashdep::setup_directory "return 0"
mock bashdep::download_url "return 1"
BASHDEP_JOBS=4
bashdep::install "https://example.com/a" "https://example.com/b" "https://example.com/c"
assert_equals 3 "$?"
}

function test_bashdep_install_jobs_one_is_sequential() {
# shellcheck disable=SC2016
mock curl 'touch "$4"'
BASHDEP_DIR="$TEST_DIR"
BASHDEP_JOBS=1
bashdep::install "https://example.com/aaa" "https://example.com/bbb" >/dev/null
local lock; lock=$(cat "$TEST_DIR/.bashdep.lock")
assert_contains "aaa" "$lock"
assert_contains "bbb" "$lock"
}

function test_bashdep_install_from_defaults_to_bashdep_file() {
printf 'https://example.com/default-tool\n' > "$TEST_DIR/.bashdep"
BASHDEP_DRY_RUN=true
Expand Down
Loading