From 8d0c08dbc97bdbb9033a7de25f779abfcf971ee9 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Fri, 13 Nov 2020 03:39:41 +0200 Subject: [PATCH] [Doc] Build docs and push to branch in CI (#1006) --- .circleci/build_docs/build_docs.sh | 13 ++++++ .circleci/build_docs/commit_docs.sh | 33 +++++++++++++++ .circleci/build_docs/install_wheels.sh | 9 +++++ .circleci/config.yml | 56 ++++++++++++++++++++++++++ .circleci/config.yml.in | 40 ++++++++++++++++++ .circleci/regenerate.py | 31 +++++++++++++- 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100755 .circleci/build_docs/build_docs.sh create mode 100755 .circleci/build_docs/commit_docs.sh create mode 100755 .circleci/build_docs/install_wheels.sh diff --git a/.circleci/build_docs/build_docs.sh b/.circleci/build_docs/build_docs.sh new file mode 100755 index 0000000000..0ce6c7ce4f --- /dev/null +++ b/.circleci/build_docs/build_docs.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex +source ./packaging/pkg_helpers.bash +export NO_CUDA_PACKAGE=1 +setup_env 0.8.0 +setup_wheel_python + +pushd docs +pip install -r requirements.txt +make html +popd + diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh new file mode 100755 index 0000000000..e125d9308f --- /dev/null +++ b/.circleci/build_docs/commit_docs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -ex + + +if [ "$2" == "" ]; then + echo call as $0 "" "" + echo where src is the built documentation and + echo branch should be "master" or "1.7" or so + exit 1 +fi + +scr=$1 +target=$2 + +set -ex +echo "committing docs from ${stc} to ${target}" + +git checkout gh-pages +rm -rf docs/$target/* +cp -r ${src}/build/html/* docs/$target +if [ $target == "master" ]; then + rm -rf docs/_static/* + cp -r ${src}/build/html/_static/* docs/_static +fi +git add docs || true +git config user.email "soumith+bot@pytorch.org" +git config user.name "pytorchbot" +# If there aren't changes, don't make a commit; push is no-op +git commit -m "auto-generating sphinx docs" || true +git push -u origin gh-pages + + diff --git a/.circleci/build_docs/install_wheels.sh b/.circleci/build_docs/install_wheels.sh new file mode 100755 index 0000000000..5ae51732b3 --- /dev/null +++ b/.circleci/build_docs/install_wheels.sh @@ -0,0 +1,9 @@ +set -ex +source ./packaging/pkg_helpers.bash +export NO_CUDA_PACKAGE=1 +setup_env 0.8.0 +setup_wheel_python +setup_pip_pytorch_version +# pytorch is already installed +pip install --no-deps ~/workspace/torchaudio* + diff --git a/.circleci/config.yml b/.circleci/config.yml index b27052977f..25ae4ce98e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -654,6 +654,46 @@ jobs: command: | .circleci/torchscript_bc_test/setup_master_envs.sh .circleci/torchscript_bc_test/validate_objects.sh + build_docs: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - checkout + - run: + name: Install pytorch-audio + command: .circleci/build_docs/install_wheels.sh + - run: + name: Build docs + command: .circleci/build_docs/build_docs.sh + - persist_to_workspace: + root: docs + paths: + - "*" + + upload_docs: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - checkout + - run: + name: Upload docs + command: | + set -ex + echo $PWD + ls ~/workspace + tag=${CIRCLE_TAG:1:5} + target=${tag:-master} + .circleci/build_docs/commit_docs.sh ~/workspace $target workflows: build: @@ -739,6 +779,22 @@ workflows: - binary_windows_conda: name: binary_windows_conda_py3.8 python_version: '3.8' + - build_docs: + name: build_docs + python_version: '3.8' + requires: + - binary_linux_wheel_py3.8 + - upload_docs: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: upload_docs + python_version: '3.8' + requires: + - build_docs unittest: jobs: - torchscript_bc_test: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 296be53463..60006ff9c7 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -654,6 +654,46 @@ jobs: command: | .circleci/torchscript_bc_test/setup_master_envs.sh .circleci/torchscript_bc_test/validate_objects.sh + build_docs: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - checkout + - run: + name: Install pytorch-audio + command: .circleci/build_docs/install_wheels.sh + - run: + name: Build docs + command: .circleci/build_docs/build_docs.sh + - persist_to_workspace: + root: docs + paths: + - "*" + + upload_docs: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - checkout + - run: + name: Upload docs + command: | + set -ex + echo $PWD + ls ~/workspace + tag=${CIRCLE_TAG:1:5} + target=${tag:-master} + .circleci/build_docs/commit_docs.sh ~/workspace $target workflows: build: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 7527dc8c69..1be0e26ffb 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -21,6 +21,8 @@ PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] +DOC_VERSION = ('linux', '3.8') + def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): w = [] @@ -30,6 +32,11 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): for python_version in PYTHON_VERSIONS: w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) + if not filter_branch: + # Build on every pull request, but upload only on nightly and tags + w += build_doc_job(None) + w += upload_doc_job('nightly') + return indent(indentation, w) @@ -67,6 +74,28 @@ def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='' return w +def build_doc_job(filter_branch): + job = { + "name": "build_docs", + "python_version": "3.8", + "requires": ["binary_linux_wheel_py3.8",], + } + + if filter_branch: + job["filters"] = gen_filter_branch_tree(filter_branch) + return [{"build_docs": job}] + +def upload_doc_job(filter_branch): + job = { + "name": "upload_docs", + "python_version": "3.8", + "requires": ["build_docs",], + } + + if filter_branch: + job["filters"] = gen_filter_branch_tree(filter_branch) + return [{"upload_docs": job}] + def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype): @@ -81,7 +110,7 @@ def generate_base_workflow(base_workflow_name, python_version, filter_branch, os if filter_branch: d["filters"] = gen_filter_branch_tree(filter_branch) - return {"binary_{os_type}_{btype}".format(os_type=os_type, btype=btype): d} + return {f"binary_{os_type}_{btype}": d} def gen_filter_branch_tree(*branches):