Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[VL] gluten-te: Enable BuildKit to support --cache-from #3964

Merged
merged 11 commits into from
Dec 11, 2023
Merged

[VL] gluten-te: Enable BuildKit to support --cache-from #3964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
0d9fe7e
fixup
zhztheplayer Dec 7, 2023
d022280
fixup
zhztheplayer Dec 7, 2023
4525097
fix
zhztheplayer Dec 7, 2023
b85a7bf
fixup
zhztheplayer Dec 8, 2023
1521d5f
fixup
zhztheplayer Dec 8, 2023
b84dca0
fixup
zhztheplayer Dec 8, 2023
847487c
fixup
zhztheplayer Dec 8, 2023
9ba0e76
fixup
zhztheplayer Dec 8, 2023
582c05a
fixup
zhztheplayer Dec 8, 2023
030365c
fixup
zhztheplayer Dec 8, 2023
5d383b0
fixup
zhztheplayer Dec 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions tools/gluten-te/centos/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ fi
BUILD_DOCKER_BUILD_ARGS=

BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg BUILDKIT_INLINE_CACHE=1"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg JDK_DEBUG_BUILD=$JDK_DEBUG_BUILD"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg GLUTEN_DEBUG_BUILD=$GLUTEN_DEBUG_BUILD"
Expand All @@ -87,13 +88,19 @@ BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-build"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --target gluten-build"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -t $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"

if [ -n "$DOCKER_CACHE_IMAGE" ]
if [ -n "$DOCKER_CACHE_REGISTRY" ]
then
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_IMAGE"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
fi

BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS $BASEDIR"

docker build $BUILD_DOCKER_BUILD_ARGS

if [ -n "$DOCKER_PUSH_REGISTRY" ]
then
docker tag "$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE" "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
fi

# EOF
22 changes: 18 additions & 4 deletions tools/gluten-te/centos/buildenv.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,15 @@ BASEDIR=$(dirname $0)

source "$BASEDIR/defaults.conf"

# Docker cache image used to speed-up builds
DOCKER_CACHE_IMAGE=${DOCKER_CACHE_IMAGE:-$DEFAULT_DOCKER_CACHE_IMAGE}
# Enable buildkit
export DOCKER_BUILDKIT=1
export BUILDKIT_PROGRESS=plain

# Docker registry used to pull pre-built images to speed-up builds
DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}

# Docker registry to push pre-built images
DOCKER_PUSH_REGISTRY=${DOCKER_PUSH_REGISTRY:-$DEFAULT_DOCKER_PUSH_REGISTRY}

# HTTP proxy
HTTP_PROXY_HOST=${HTTP_PROXY_HOST:-$DEFAULT_HTTP_PROXY_HOST}
Expand Down Expand Up @@ -56,6 +63,7 @@ fi
BUILDENV_DOCKER_BUILD_ARGS=

BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg BUILDKIT_INLINE_CACHE=1"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg TIMEZONE=$TIMEZONE"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg OS_IMAGE=$OS_IMAGE --build-arg OS_VERSION=$OS_VERSION"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg MAVEN_MIRROR_URL=$MAVEN_MIRROR_URL"
Expand All @@ -65,13 +73,19 @@ BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-b
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --target gluten-buildenv"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -t $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"

if [ -n "$DOCKER_CACHE_IMAGE" ]
if [ -n "$DOCKER_CACHE_REGISTRY" ]
then
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_IMAGE"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
fi

BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS $BASEDIR"

docker build $BUILDENV_DOCKER_BUILD_ARGS

if [ -n "$DOCKER_PUSH_REGISTRY" ]
then
docker tag "$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE" "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
fi

# EOF
7 changes: 5 additions & 2 deletions tools/gluten-te/centos/defaults.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,11 @@ DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
# Build will result in this image
DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build

# Docker cache image used to speed-up builds
DEFAULT_DOCKER_CACHE_IMAGE=
# Docker registry used to pull pre-built images to speed-up builds
DEFAULT_DOCKER_CACHE_REGISTRY=

# Docker registry to push pre-built images
DEFAULT_DOCKER_PUSH_REGISTRY=

## For cbash.sh

Expand Down
11 changes: 9 additions & 2 deletions tools/gluten-te/ubuntu/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ fi
BUILD_DOCKER_BUILD_ARGS=

BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg BUILDKIT_INLINE_CACHE=1"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg JDK_DEBUG_BUILD=$JDK_DEBUG_BUILD"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --build-arg GLUTEN_DEBUG_BUILD=$GLUTEN_DEBUG_BUILD"
Expand All @@ -87,13 +88,19 @@ BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-build"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --target gluten-build"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS -t $DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"

if [ -n "$DOCKER_CACHE_IMAGE" ]
if [ -n "$DOCKER_CACHE_REGISTRY" ]
then
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_IMAGE"
BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
fi

BUILD_DOCKER_BUILD_ARGS="$BUILD_DOCKER_BUILD_ARGS $BASEDIR"

docker build $BUILD_DOCKER_BUILD_ARGS

if [ -n "$DOCKER_PUSH_REGISTRY" ]
then
docker tag "$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE" "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
fi

# EOF
22 changes: 18 additions & 4 deletions tools/gluten-te/ubuntu/buildenv.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,15 @@ BASEDIR=$(dirname $0)

source "$BASEDIR/defaults.conf"

# Docker cache image used to speed-up builds
DOCKER_CACHE_IMAGE=${DOCKER_CACHE_IMAGE:-$DEFAULT_DOCKER_CACHE_IMAGE}
# Enable buildkit
export DOCKER_BUILDKIT=1
export BUILDKIT_PROGRESS=plain

# Docker registry used to pull pre-built images to speed-up builds
DOCKER_CACHE_REGISTRY=${DOCKER_CACHE_REGISTRY:-$DEFAULT_DOCKER_CACHE_REGISTRY}

# Docker registry to push pre-built images
DOCKER_PUSH_REGISTRY=${DOCKER_PUSH_REGISTRY:-$DEFAULT_DOCKER_PUSH_REGISTRY}

# HTTP proxy
HTTP_PROXY_HOST=${HTTP_PROXY_HOST:-$DEFAULT_HTTP_PROXY_HOST}
Expand Down Expand Up @@ -53,6 +60,7 @@ fi
BUILDENV_DOCKER_BUILD_ARGS=

BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg BUILDKIT_INLINE_CACHE=1"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg TIMEZONE=$TIMEZONE"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg OS_IMAGE=$OS_IMAGE"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --build-arg MAVEN_MIRROR_URL=$MAVEN_MIRROR_URL"
Expand All @@ -62,13 +70,19 @@ BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-b
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --target gluten-buildenv"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS -t $DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"

if [ -n "$DOCKER_CACHE_IMAGE" ]
if [ -n "$DOCKER_CACHE_REGISTRY" ]
then
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_IMAGE"
BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
fi

BUILDENV_DOCKER_BUILD_ARGS="$BUILDENV_DOCKER_BUILD_ARGS $BASEDIR"

docker build $BUILDENV_DOCKER_BUILD_ARGS

if [ -n "$DOCKER_PUSH_REGISTRY" ]
then
docker tag "$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE" "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_TARGET_IMAGE_BUILDENV_WITH_OS_IMAGE"
fi

# EOF
7 changes: 5 additions & 2 deletions tools/gluten-te/ubuntu/defaults.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,11 @@ DEFAULT_DOCKER_TARGET_IMAGE_BUILDENV=gluten-te/gluten-buildenv
# Build will result in this image
DEFAULT_DOCKER_TARGET_IMAGE_BUILD=gluten-te/gluten-build

# Docker cache image used to speed-up builds
DEFAULT_DOCKER_CACHE_IMAGE=
# Docker registry used to pull pre-built images to speed-up builds
DEFAULT_DOCKER_CACHE_REGISTRY=

# Docker registry to push pre-built images
DEFAULT_DOCKER_PUSH_REGISTRY=

## For tpc.sh

Expand Down
19 changes: 14 additions & 5 deletions tools/gluten-te/ubuntu/tpc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,26 @@ JDWP_SERVER_PORT=${JDWP_SERVER_PORT:-$DEFAULT_JDWP_SERVER_PORT}

TPC_DOCKER_BUILD_ARGS=
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --ulimit nofile=8192:8192"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg BUILDKIT_INLINE_CACHE=1"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE=$DOCKER_TARGET_IMAGE_BUILD_WITH_OS_IMAGE"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --build-arg BUILD_BACKEND_TYPE=$BUILD_BACKEND_TYPE"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS -f $BASEDIR/dockerfile-tpc"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --target $DOCKER_BUILD_TARGET_NAME"
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS -t $DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"

if [ -n "$DOCKER_CACHE_REGISTRY" ]
then
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
fi

TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS $BASEDIR"

if [ -n "$DOCKER_CACHE_IMAGE" ]
docker build $TPC_DOCKER_BUILD_ARGS

if [ -n "$DOCKER_PUSH_REGISTRY" ]
then
TPC_DOCKER_BUILD_ARGS="$TPC_DOCKER_BUILD_ARGS --cache-from $DOCKER_CACHE_IMAGE"
docker tag "$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE" "$DOCKER_PUSH_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
docker push "$DOCKER_PUSH_REGISTRY/$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE"
fi

TPC_DOCKER_RUN_ARGS=
Expand Down Expand Up @@ -108,7 +118,7 @@ then
JAVA_ARGS="$JAVA_ARGS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$JDWP_SERVER_PORT"
fi
JAVA_ARGS="$JAVA_ARGS $EXTRA_JAVA_OPTIONS"
JAVA_ARGS="$JAVA_ARGS -cp /opt/gluten/tools/gluten-it/target/gluten-it-1.0-SNAPSHOT-jar-with-dependencies.jar"
JAVA_ARGS="$JAVA_ARGS -cp '/opt/gluten/tools/gluten-it/package/target/lib/*'"
JAVA_ARGS="$JAVA_ARGS io.glutenproject.integration.tpc.Tpc $TPC_CMD_ARGS"

BASH_ARGS=
Expand All @@ -122,7 +132,6 @@ else
BASH_ARGS="java $JAVA_ARGS"
fi

docker build $TPC_DOCKER_BUILD_ARGS
docker run $TPC_DOCKER_RUN_ARGS $DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE "$BASH_ARGS"
docker run $TPC_DOCKER_RUN_ARGS "$DOCKER_SELECTED_TARGET_IMAGE_TPC_WITH_OS_IMAGE" "$BASH_ARGS"

# EOF
Loading