Skip to content

Commit 47148b3

Browse files
mateuszlewkomateuszlewko
authored
Configure Terrafrom to build all required artifacts in the new GCP project for releases (#4842)
* Remove / change obsolete comments * WIP * Add initial Terraform modules * Add worker pool * Add trigger_schedule_account module * Add docker_registry module * Remove empty modules * Add xla_docker_build module * Add version and nightly builds * Fix any errors that prevented from working * Add tpu-pytorch project configuration * Debugging * Clean-up * Add libcudnn8 deps for old cuda versions * Minor * Minor * Remove builds for Python 3.7 * Add build step to fetch ansible config at specific branch * Removed experimental dockerfile and cloudbuild file * Use current branch for all ansible configs * Fix a bug and add back triggers for Py3.7 * Update wheels destination * Add 'Rename and collect wheels' task * Fix docker copy statement * Add 'docker_image' suffix to build steps * Fix renaming wheels, and passing Ansible variables * Fix build arg flag name * Debugging * Put ARG just before run statement * Wrap ansible vars in single quotes * Wrap in double quotes * Debugging * Fix dockerfiles * Add a TODO * Format TF files * Fix scheduler job url * Fix public read access for docker repo * Specify project IAM admins * Configure IAMs by adding admins * Configure IAMs for remote build * Add iam member resource for project_remote_build_writers * Add trigger for bazel * Remove bazel trigger
1 parent b248e21 commit 47148b3

File tree

35 files changed

+1038
-330
lines changed

35 files changed

+1038
-330
lines changed

‎docker/experimental/Dockerfile‎

Lines changed: 0 additions & 141 deletions
This file was deleted.

‎docker/experimental/ansible/Dockerfile‎

Lines changed: 15 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,54 +3,31 @@ ARG debian_version=buster
33

44
FROM python:${python_version}-${debian_version} AS build
55

6+
WORKDIR /ansible
67
RUN pip install ansible
7-
88
COPY . /ansible
9-
WORKDIR /ansible
109

11-
ARG arch=amd64
12-
ARG accelerator=tpu
13-
ARG cuda_version=11.8
14-
ARG pytorch_git_rev=HEAD
15-
ARG xla_git_rev=HEAD
16-
ARG package_version
17-
18-
RUN ansible-playbook -vvv playbook.yaml -e \
19-
"stage=build \
20-
arch=${arch} \
21-
accelerator=${accelerator} \
22-
cuda_version=${cuda_version} \
23-
pytorch_git_rev=${pytorch_git_rev} \
24-
xla_git_rev=${xla_git_rev} \
25-
package_version=${package_version}"
10+
ARG ansible_vars
11+
RUN ansible-playbook -vvv playbook.yaml -e "stage=build" -e "${ansible_vars}"
2612

2713
FROM python:${python_version}-${debian_version} AS release
2814

2915
WORKDIR /ansible
16+
RUN pip install ansible
3017
COPY . /ansible
3118

32-
ARG arch=amd64
33-
ARG accelerator=tpu
34-
ARG cuda_version=11.8
35-
ARG pytorch_git_rev=HEAD
36-
ARG xla_git_rev=HEAD
19+
ARG ansible_vars
20+
RUN ansible-playbook -vvv playbook.yaml -e "stage=release" -e "${ansible_vars}" --tags "install_deps"
3721

38-
RUN pip install ansible
39-
RUN ansible-playbook -vvv playbook.yaml -e \
40-
"stage=release \
41-
arch=${arch} \
42-
accelerator=${accelerator} \
43-
cuda_version=${cuda_version} \
44-
pytorch_git_rev=${pytorch_git_rev} \
45-
xla_git_rev=${xla_git_rev} \
46-
" --tags "install_deps"
47-
48-
WORKDIR /dist
49-
COPY --from=build /src/pytorch/dist/*.whl .
50-
COPY --from=build /src/pytorch/xla/dist/*.whl .
51-
52-
RUN echo "Installing the following wheels" && ls /dist/*.whl
22+
WORKDIR /tmp/wheels
23+
COPY --from=build /src/pytorch/dist/*.whl ./
24+
COPY --from=build /src/pytorch/xla/dist/*.whl ./
25+
26+
RUN echo "Installing the following wheels" && ls *.whl
5327
RUN pip install *.whl
5428

5529
WORKDIR /
56-
RUN rm -rf /ansible
30+
31+
RUN rm -rf /ansible /tmp/wheels
32+
COPY --from=build /dist/*.whl /dist/
33+

‎docker/experimental/ansible/config/apt.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ apt:
1818
- less
1919

2020
build_cuda:
21-
- "cuda-libraries-{{ cuda_version | replace('.', '-') }}"
22-
- "cuda-toolkit-{{ cuda_version | replace('.', '-') }}"
23-
- "cuda-minimal-build-{{ cuda_version | replace('.', '-') }}"
21+
- "cuda-libraries-{{ cuda_version | replace('.', '-') }}"
22+
- "cuda-toolkit-{{ cuda_version | replace('.', '-') }}"
23+
- "cuda-minimal-build-{{ cuda_version | replace('.', '-') }}"
2424
- "{{ cuda_deps['libcudnn'][cuda_version] }}"
2525
- "{{ cuda_deps['libcudnn-dev'][cuda_version] }}"
2626

@@ -42,8 +42,8 @@ apt:
4242
- patch
4343

4444
release_cuda:
45-
- "cuda-libraries-{{ cuda_version | replace('.', '-') }}"
46-
- "cuda-minimal-build-{{ cuda_version | replace('.', '-') }}"
45+
- "cuda-libraries-{{ cuda_version | replace('.', '-') }}"
46+
- "cuda-minimal-build-{{ cuda_version | replace('.', '-') }}"
4747
- "{{ cuda_deps['libcudnn'][cuda_version] }}"
4848

4949
# Specify objects with string fields `url` and `keyring`.

‎docker/experimental/ansible/config/cuda_deps.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
# Versions of cuda dependencies for given cuda versions.
22
# Note: wrap version in quotes to ensure they're treated as strings.
33
cuda_deps:
4+
# List all libcudnn8 versions with `apt list -a libcudnn8`
45
libcudnn:
56
"11.8": libcudnn8=8.8.0.121-1+cuda11.8
7+
"11.7": libcudnn8=8.5.0.96-1+cuda11.7
8+
"11.2": libcudnn8=8.1.1.33-1+cuda11.2
69
libcudnn-dev:
710
"11.8": libcudnn8-dev=8.8.0.121-1+cuda11.8
11+
"11.7": libcudnn8-dev=8.5.0.96-1+cuda11.7
12+
"11.2": libcudnn8-dev=8.1.1.33-1+cuda11.2

‎docker/experimental/ansible/config/vars.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,6 @@ cuda_version: "11.8"
55
llvm_debian_repo: buster
66
clang_version: 10
77
# PyTorch and PyTorch/XLA wheel versions.
8-
package_version: 2.0
8+
package_version: 2.0
9+
# If set to true, wheels will be renamed to $WHEEL_NAME-nightly-cp38-cp38-linux_x86_64.whl.
10+
nightly_release: false

‎docker/experimental/ansible/development.Dockerfile‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ RUN pip install ansible
1111
COPY . /ansible
1212
WORKDIR /ansible
1313

14-
ARG arch=amd64
15-
ARG accelerator=tpu
14+
# List Asnible tasks to apply for the dev image.
15+
ENV TAGS="bazel,configure_env,install_deps"
1616

17-
RUN ansible-playbook playbook.yaml -e "stage=build arch=${arch} accelerator=${accelerator}" --skip-tags "fetch_srcs,build_srcs"
18-
RUN ansible-playbook playbook.yaml -e "stage=release arch=${arch} accelerator=${accelerator}" --skip-tags "fetch_srcs,build_srcs"
17+
ARG ansible_vars
18+
RUN ansible-playbook playbook.yaml -e "stage=build" -e "${ansible_vars}" --tags "${TAGS}"
19+
RUN ansible-playbook playbook.yaml -e "stage=release" -e "${ansible_vars}" --tags "${TAGS}"

‎docker/experimental/ansible/playbook.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
- apt.yaml
3434
- pip.yaml
3535
- env.yaml
36-
tags: always # Execute this task even with `--skip-tags` is used.
36+
tags: always # Execute this task even with `--skip-tags` or `--tags` is used.
3737

3838
roles:
3939
- role: bazel

‎docker/experimental/ansible/roles/build_srcs/tasks/main.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,49 @@
4040
name: "{{ xla_wheels.files | map(attribute='path') }}"
4141
state: "forcereinstall"
4242

43+
- name: Create a temp directory for collecting wheels
44+
ansible.builtin.file:
45+
path: /tmp/staging-wheels
46+
state: directory
47+
mode: '0755'
48+
49+
- name: Create /dist directory for exported wheels
50+
ansible.builtin.file:
51+
path: /dist
52+
state: directory
53+
mode: '0755'
54+
55+
- name: Rename and append +YYYYMMDD suffix to nightly wheels
56+
ansible.builtin.shell: |
57+
pushd /tmp/staging-wheels
58+
cp {{ item.dir }}/*.whl .
59+
rename -v "s/^{{ item.prefix }}-(.*?)-cp/{{ item.prefix }}-nightly-cp/" *.whl
60+
mv /tmp/staging-wheels/* /dist/
61+
popd
62+
63+
rename -v "s/^{{ item.prefix }}-(.*?)-cp/{{ item.prefix }}-nightly+$(date -u +%Y%m%d)-cp/" *.whl
64+
args:
65+
executable: /bin/bash
66+
chdir: "{{ item.dir }}"
67+
loop:
68+
- { dir: "{{ (src_root, 'pytorch/dist') | path_join }}", prefix: "torch" }
69+
- { dir: "{{ (src_root, 'pytorch/xla/dist') | path_join }}", prefix: "torch_xla" }
70+
when: nightly_release
71+
72+
- name: Copy wheels to /dist
73+
ansible.builtin.shell: "cp {{ item }}/*.whl /dist"
74+
args:
75+
executable: /bin/bash
76+
loop:
77+
- "{{ (src_root, 'pytorch/dist') | path_join }}"
78+
- "{{ (src_root, 'pytorch/xla/dist') | path_join }}"
79+
80+
- name: Delete temp directory
81+
ansible.builtin.file:
82+
path: /tmp/staging-wheels
83+
state: absent
84+
mode: '0755'
85+
4386
- name: "Tests"
4487
include_tasks: tests.yaml
4588
tags:

0 commit comments

Comments
 (0)