Skip to content
Permalink
Browse files
HBASE-26828 Increase the concurrency when running UTs in pre commit j…
…ob (#4213)

Signed-off-by: Yulin Niu <niuyulin@apache.org>
  • Loading branch information
Apache9 committed Mar 14, 2022
1 parent d542f47 commit a49d147d49b079c3974e9a29fd1a6d861ca6b525
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 2 deletions.
@@ -45,7 +45,10 @@ pipeline {
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"

// set build parallel
BUILD_THREAD = 4
SUREFIRE_FIRST_PART_FORK_COUNT = '0.5C'
SUREFIRE_SECOND_PART_FORK_COUNT = '0.5C'
// a global view of paths. parallel stages can land on the same host concurrently, so each
// stage works in its own subdirectory. there is an "output" under each of these
// directories, which we retrieve after the build is complete.
@@ -119,6 +119,18 @@ function personality_parse_args
delete_parameter "${i}"
ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
;;
--build-thread=*
delete_parameter "${i}"
BUILD_THREAD=${i#*=}
;;
--surefire-first-part-fork-count=*
delete_parameter "${i}"
SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
;;
--surefire-second-part-fork-count=*
delete_parameter "${i}"
SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
;;
esac
done
}
@@ -144,7 +156,13 @@ function personality_modules
# At a few points, hbase modules can run build, test, etc. in parallel
# Let it happen. Means we'll use more CPU but should be for short bursts.
# https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
extra="--threads=2 -DHBasePatchProcess"
if [[ -n "${BUILD_THREAD}" ]]; then
extra="--threads=${BUILD_THREAD}"
else
extra="--threads=2"
fi

extra="${extra} -DHBasePatchProcess"
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
extra="${extra} -Dhttps.protocols=TLSv1.2"
fi
@@ -232,6 +250,15 @@ function personality_modules
extra="${extra} -Dbuild.id=${BUILD_ID}"
fi

# set forkCount
if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
fi

if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
fi

# If the set of changed files includes CommonFSUtils then add the hbase-server
# module to the set of modules (if not already included) to be tested
for f in "${CHANGED_FILES[@]}"
@@ -147,6 +147,16 @@ YETUS_ARGS+=("--github-use-emoji-vote")
if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
fi
# pass build parallelism in
if [[ -n "${BUILD_THREAD}" ]]; then
YETUS_ARGS+=("--build-thread=${BUILD_THREAD}")
fi
if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
YETUS_ARGS+=("--surefire-first-part-fork-count=${SUREFIRE_FIRST_PART_FORK_COUNT}")
fi
if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
YETUS_ARGS+=("--surefire-second-part-fork-count=${SUREFIRE_SECOND_PART_FORK_COUNT}")
fi

echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

0 comments on commit a49d147

Please sign in to comment.