From d0b15e299be39749f9dea2a25b90d22c47f8bb5c Mon Sep 17 00:00:00 2001 From: Philipp Burckhardt Date: Tue, 4 Aug 2026 23:34:13 -0500 Subject: [PATCH] build: replace `uniq` with `sort -u` to deduplicate directories in CI workflows The `uniq` command only removes adjacent duplicates, so when `git diff` output produces the same parent directory in non-adjacent positions (e.g., files in root interleaved with subdirectory files), duplicate directory entries survive. This causes test files, benchmarks, and coverage runs to execute multiple times for the same package. Replace `uniq` with `sort -u` in all affected workflow YAML files and their companion shell scripts. Additionally, add a deduplication step after merging changed directories with dependent directories in the affected tests script. Closes: https://github.com/stdlib-js/metr-issue-tracker/issues/1113 Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/run_affected_benchmarks.yml | 2 +- .github/workflows/run_affected_tests.yml | 2 +- .github/workflows/run_tests_coverage.yml | 2 +- .github/workflows/run_tests_coverage_pr.yml | 2 +- .github/workflows/scripts/run_affected_benchmarks/run | 2 +- .github/workflows/scripts/run_affected_tests/run | 5 ++++- .github/workflows/scripts/run_tests_coverage/run | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_affected_benchmarks.yml b/.github/workflows/run_affected_benchmarks.yml index 4d8d51803e3f..42381a7fbec5 100644 --- a/.github/workflows/run_affected_benchmarks.yml +++ b/.github/workflows/run_affected_benchmarks.yml @@ -131,7 +131,7 @@ jobs: files=$(git diff --diff-filter=AM --name-only ${{ github.event.before }} ${{ github.event.after }}) fi fi - directories=$(for file in $files; do dirname $file; done | uniq | tr '\n' ' ' | sed 's/ $//') + directories=$(for file in $files; do dirname $file; done | sort -u | tr '\n' ' ' | sed 's/ $//') echo "directories=${directories}" >> $GITHUB_OUTPUT # Get list of changed directories from workflow dispatch event: diff --git a/.github/workflows/run_affected_tests.yml b/.github/workflows/run_affected_tests.yml index cad27d1029db..307a0a7e1b47 100644 --- a/.github/workflows/run_affected_tests.yml +++ b/.github/workflows/run_affected_tests.yml @@ -163,7 +163,7 @@ jobs: files=$(git diff --diff-filter=AM --name-only ${{ github.event.before }} ${{ github.event.after }}) fi fi - directories=$(for file in $files; do dirname $file; done | uniq | tr '\n' ' ' | sed 's/ $//') + directories=$(for file in $files; do dirname $file; done | sort -u | tr '\n' ' ' | sed 's/ $//') echo "directories=${directories}" >> $GITHUB_OUTPUT # Get list of changed directories from workflow dispatch event: diff --git a/.github/workflows/run_tests_coverage.yml b/.github/workflows/run_tests_coverage.yml index 37392e29e0d4..071821bb9d15 100644 --- a/.github/workflows/run_tests_coverage.yml +++ b/.github/workflows/run_tests_coverage.yml @@ -134,7 +134,7 @@ jobs: else files=$(git diff --diff-filter=AM --name-only ${{ github.event.before }} ${{ github.event.after }}) fi - directories=$(for file in $files; do dirname $file; done | uniq | tr '\n' ' ' | sed 's/ $//') + directories=$(for file in $files; do dirname $file; done | sort -u | tr '\n' ' ' | sed 's/ $//') echo "directories=${directories}" >> $GITHUB_OUTPUT # Get list of changed directories from workflow dispatch event: diff --git a/.github/workflows/run_tests_coverage_pr.yml b/.github/workflows/run_tests_coverage_pr.yml index 2a6beab9fd80..0394da521c2d 100644 --- a/.github/workflows/run_tests_coverage_pr.yml +++ b/.github/workflows/run_tests_coverage_pr.yml @@ -123,7 +123,7 @@ jobs: # Get changed files using git diff against base branch: git fetch origin ${{ github.base_ref }} --depth=1 files=$(git diff --diff-filter=AM --name-only origin/${{ github.base_ref }}...HEAD) - directories=$(for file in $files; do dirname $file; done | uniq | tr '\n' ' ' | sed 's/ $//') + directories=$(for file in $files; do dirname $file; done | sort -u | tr '\n' ' ' | sed 's/ $//') echo "directories=${directories}" >> $GITHUB_OUTPUT # Exit early if non-package directories are changed: diff --git a/.github/workflows/scripts/run_affected_benchmarks/run b/.github/workflows/scripts/run_affected_benchmarks/run index eec70bfa3f88..765a4045a002 100755 --- a/.github/workflows/scripts/run_affected_benchmarks/run +++ b/.github/workflows/scripts/run_affected_benchmarks/run @@ -110,7 +110,7 @@ main() { changed=$(echo "${changed}" | tr ' ' '\n' | grep '^lib/node_modules/@stdlib') || true # Find unique package directories: - directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?$//' | uniq) + directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?$//' | sort -u) if [ -z "${directories}" ]; then echo 'No packages to run benchmarks for.' >&2 diff --git a/.github/workflows/scripts/run_affected_tests/run b/.github/workflows/scripts/run_affected_tests/run index cb1add9e441f..fe58584b8a9d 100755 --- a/.github/workflows/scripts/run_affected_tests/run +++ b/.github/workflows/scripts/run_affected_tests/run @@ -110,7 +110,7 @@ main() { changed=$(echo "${changed}" | tr ' ' '\n' | grep '^lib/node_modules/@stdlib') || true # Find unique package directories: - directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?$//' | uniq) + directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?$//' | sort -u) if [ -z "${directories}" ]; then echo 'No packages to test.' >&2 @@ -141,6 +141,9 @@ main() { directories="${directories} ${required_by}" fi + # Deduplicate the merged list of directories: + directories=$(echo "${directories}" | tr ' ' '\n' | sort -u) + # Build native add-ons for packages (if applicable): for pkg in ${packages}; do if [ -f "lib/node_modules/${pkg}/binding.gyp" ]; then diff --git a/.github/workflows/scripts/run_tests_coverage/run b/.github/workflows/scripts/run_tests_coverage/run index a86cce271eb0..35a6ab95799c 100755 --- a/.github/workflows/scripts/run_tests_coverage/run +++ b/.github/workflows/scripts/run_tests_coverage/run @@ -129,7 +129,7 @@ main() { grep -v '^lib/node_modules/@stdlib/_tools') || true # Find unique package directories: - directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?\/?$//' | uniq) + directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(benchmark|bin|data|docs|etc|examples|include|lib|scripts|src|test)(\/.*)?\/?$//' | sort -u) if [ -z "${directories}" ]; then echo 'No packages to test.' >&2