#85 aggregate-add in threads

steps/aggregate-add.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# The MIT License (MIT)
+#
+# Copyright (c) 2021-2023 Yegor Bugayenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+set -e
+set -o pipefail
+
+repo=$1
+dir=$2
+pos=$3
+total=$4
+
+start=$(date +%s%N)
+
+csvs=$(find "${dir}" -name '*.csv' -maxdepth 1 -exec basename {} \;)
+
+total=$(echo "${csvs}" | wc -l | xargs)
+echo "${csvs}" | while IFS= read -r csv; do
+    while IFS= read -r t; do
+        printf '%s,%s\n' "${reop}" "${t}" >> "${TARGET}/data/${csv}"
+    done < "${dir}/${csv}"
+done
+
+echo "${total} metrics added to the CSV aggregate of ${repo} (${pos}/${total})$("${LOCAL}/help/tdiff.sh" "${start}")"

steps/aggregate.sh

@@ -53,14 +53,19 @@ echo "${all}" | while IFS= read -r a; do
 done
 printf "\n" >> "${TARGET}/data/all.csv"
 
-find "${TARGET}/data" -maxdepth 2 -mindepth 2 -type d -print | while IFS= read -r d; do
-    r=$(realpath --relative-to="${TARGET}/data" "$d" )
-    find "${d}" -name '*.csv' -maxdepth 1 -exec basename {} \; | while IFS= read -r csv; do
-        while IFS= read -r t; do
-            echo "${r},${t}" >> "${TARGET}/data/${csv}"
-        done < "${d}/${csv}"
-    done
-    echo "${r} metrics added to the CSV aggregate"
-done
+jobs=${TARGET}/jobs/aggregate-add-jobs.txt
+rm -rf "${jobs}"
+mkdir -p "$(dirname "${jobs}")"
+touch "${jobs}"
+declare -i repo=0
+sh="$(dirname "$0")/aggregate-add.sh"
+repos=$(find "${TARGET}/data" -maxdepth 2 -mindepth 2 -type d -print)
+echo "${repos}" | while IFS= read -r d; do
+    r=$(realpath --relative-to="${TARGET}/data" "${d}" )
+    repo=$((repo+1))
+    printf "%s %s %s %s %s\n" "${sh@Q}" "${r@Q}" "${d@Q}" "${repo@Q}" "${total@Q}" >> "${jobs}"
+ done
+"${LOCAL}/help/parallel.sh" "${jobs}"
+wait
 
 echo "All metrics aggregated in ${total} repositories in $(nproc) threads$("${LOCAL}/help/tdiff.sh" "${start}")"