Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

factorize cgroups detection (for proper jobs/load settings) #1764

Merged
merged 1 commit into from
Apr 7, 2024
Merged

factorize cgroups detection (for proper jobs/load settings) #1764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 2 additions & 20 deletions .ci/build_script.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,6 @@ fi
mkdir -p "${CCACHE_DIR}"
echo "using cache dir: ${CCACHE_DIR}"

PROCS="$(getconf _NPROCESSORS_ONLN)"
if [[ -r /sys/fs/cgroup/cpu/cpu.shares ]]; then
CG_PROCS=$(($(cat /sys/fs/cgroup/cpu/cpu.shares) / 1024))
echo "running under cgroups, allocated vs physical cores: ${CG_PROCS} / ${PROCS}"
PARALLEL_JOBS=$((CG_PROCS + (CG_PROCS + 1) / 2))
PARALLEL_LOAD="${PROCS}"
else
PARALLEL_JOBS=$((PROCS + (PROCS + 1) / 2))
PARALLEL_LOAD="${PROCS}"
fi
echo "PARALLEL_JOBS: ${PARALLEL_JOBS}"
echo "PARALLEL_LOAD: ${PARALLEL_LOAD}"

travis_retry make fetchthirdparty TARGET=

docker-make() {
Expand All @@ -43,15 +30,10 @@ docker-make() {
/bin/bash -c "$(printf '%s && ' "${cmdlist[@]}")true"
}

makeargs=(
PARALLEL_JOBS="${PARALLEL_JOBS}"
PARALLEL_LOAD="${PARALLEL_LOAD}"
TARGET="${TARGET}"
)
if [[ -z "${DOCKER_IMG}" ]]; then
'./.ci/cache_restore_post.sh'
trap './.ci/cache_save_pre.sh' EXIT
make "${makeargs[@]}" "$@"
make TARGET="${TARGET}" "$@"
else
docker-make "${makeargs[@]}" VERBOSE=1 "$@"
docker-make TARGET="${TARGET}" VERBOSE=1 "$@"
fi
3 changes: 0 additions & 3 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,6 @@ jobs:
id: build
run: |
export MACOSX_DEPLOYMENT_TARGET=11;
PROCS="$(getconf _NPROCESSORS_ONLN)"
export PARALLEL_JOBS="$((PROCS + (PROCS + 1) / 2))"
export PARALLEL_LOAD="${PROCS}"
make fetchthirdparty && make

- name: Build cache pre-save
Expand Down
19 changes: 17 additions & 2 deletions Makefile.defs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,25 @@
SHELL = env PATH='$(PATH)' bash

ifeq (,$(PARALLEL_JOBS))
PARALLEL_JOBS := $(or $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS))),$(shell getconf _NPROCESSORS_ONLN))
PARALLEL_JOBS := $(or $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS))))
endif
ifeq (,$(PARALLEL_LOAD))
PARALLEL_LOAD := $(or $(patsubst -l%,%,$(filter -l%,$(MAKEFLAGS))),$(shell getconf _NPROCESSORS_ONLN))
PARALLEL_LOAD := $(or $(patsubst -l%,%,$(filter -l%,$(MAKEFLAGS))))
endif
ifeq (,$(and $(PARALLEL_JOBS),$(PARALLEL_LOAD)))
CORES := $(shell getconf _NPROCESSORS_ONLN)
ALLOCATED_CORES := $(CORES)
# Detect when running under cgroups, so we don't try
# to run with `make/ninja -j36 …` on some CI workers.
ifneq (,$(wildcard /sys/fs/cgroup/cpu/cpu.shares))
ALLOCATED_CORES := $(shell echo $$(($$(cat /sys/fs/cgroup/cpu/cpu.shares) / 1024)))
endif
ifeq (,$(PARALLEL_JOBS))
PARALLEL_JOBS := $(if $(CI),$(shell echo $$(($(ALLOCATED_CORES) + ($(ALLOCATED_CORES) + 1) / 2))),$(ALLOCATED_CORES))
endif
ifeq (,$(PARALLEL_LOAD))
PARALLEL_LOAD := $(CORES)
endif
endif
export PARALLEL_JOBS
export PARALLEL_LOAD
Expand Down