Skip to content

0.3.0 packaging #221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 8, 2019
Merged

0.3.0 packaging #221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
45d1da5
Rewrite our packaging from scratch (#217)
ezyang Aug 7, 2019
a0cdfbf
Setup cuda_suffix (#218)
ezyang Aug 7, 2019
6ba50a0
Add uploading support (#220)
ezyang Aug 8, 2019
1e18469
Finish up and set parameters
ezyang Aug 8, 2019
8cc350a
Updates from vision
ezyang Aug 8, 2019
bf88aef
Go time
ezyang Aug 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
308 changes: 308 additions & 0 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,308 @@
version: 2.1

# How to test the Linux jobs:
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
# Job names are 'name:' key.

binary_common: &binary_common
parameters:
# Edit these defaults to do a release
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: "0.3.0"
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: "1.2.0"
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.7)"
type: string
unicode_abi:
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
type: string
default: ""
environment:
PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: cpu

jobs:
circleci_consistency:
docker:
- image: circleci/python:3.7
steps:
- checkout
- run:
command: |
pip install --user --progress-bar off jinja2
python .circleci/regenerate.py
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

binary_linux_wheel:
<<: *binary_common
docker:
- image: "soumith/manylinux-cuda100"
resource_class: 2xlarge+
steps:
- checkout
- run: packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"

binary_linux_conda:
<<: *binary_common
docker:
- image: "soumith/conda-cuda"
resource_class: 2xlarge+
steps:
- checkout
- run: packaging/build_conda.sh
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
root: /opt/conda/conda-bld/linux-64
paths:
- "*"

binary_macos_wheel:
<<: *binary_common
macos:
xcode: "9.0"
steps:
- checkout
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
packaging/build_wheel.sh
- store_artifacts:
path: dist
- persist_to_workspace:
root: dist
paths:
- "*"

binary_macos_conda:
<<: *binary_common
macos:
xcode: "9.0"
steps:
- checkout
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
packaging/build_conda.sh
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3/conda-bld/osx-64
paths:
- "*"

# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set +x
anaconda login \
--username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
--password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
set -x
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force

# Requires org-member context
binary_wheel_upload:
docker:
- image: circleci/python:3.7
steps:
- attach_workspace:
at: ~/workspace
- checkout
- run:
command: |
pip install --user awscli
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
done

workflows:
build:
jobs:
- circleci_consistency

- binary_linux_wheel:
name: nightly_binary_linux_wheel_py2.7
python_version: "2.7"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7
- binary_linux_wheel:
name: nightly_binary_linux_wheel_py2.7_unicode
python_version: "2.7"
unicode_abi: "1"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7_unicode_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7_unicode
- binary_linux_wheel:
name: nightly_binary_linux_wheel_py3.5
python_version: "3.5"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.5_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.5
- binary_linux_wheel:
name: nightly_binary_linux_wheel_py3.6
python_version: "3.6"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.6_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.6
- binary_linux_wheel:
name: nightly_binary_linux_wheel_py3.7
python_version: "3.7"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.7_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.7
- binary_macos_wheel:
name: nightly_binary_macos_wheel_py2.7
python_version: "2.7"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py2.7_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py2.7
- binary_macos_wheel:
name: nightly_binary_macos_wheel_py2.7_unicode
python_version: "2.7"
unicode_abi: "1"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py2.7_unicode_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py2.7_unicode
- binary_macos_wheel:
name: nightly_binary_macos_wheel_py3.5
python_version: "3.5"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.5_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.5
- binary_macos_wheel:
name: nightly_binary_macos_wheel_py3.6
python_version: "3.6"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.6_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.6
- binary_macos_wheel:
name: nightly_binary_macos_wheel_py3.7
python_version: "3.7"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.7_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.7
- binary_linux_conda:
name: nightly_binary_linux_conda_py2.7
python_version: "2.7"
- binary_conda_upload:
name: nightly_binary_linux_conda_py2.7_upload
context: org-member
requires:
- nightly_binary_linux_conda_py2.7
- binary_linux_conda:
name: nightly_binary_linux_conda_py3.5
python_version: "3.5"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.5_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.5
- binary_linux_conda:
name: nightly_binary_linux_conda_py3.6
python_version: "3.6"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.6_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.6
- binary_linux_conda:
name: nightly_binary_linux_conda_py3.7
python_version: "3.7"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.7_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.7
- binary_macos_conda:
name: nightly_binary_macos_conda_py2.7
python_version: "2.7"
- binary_conda_upload:
name: nightly_binary_macos_conda_py2.7_upload
context: org-member
requires:
- nightly_binary_macos_conda_py2.7
- binary_macos_conda:
name: nightly_binary_macos_conda_py3.5
python_version: "3.5"
- binary_conda_upload:
name: nightly_binary_macos_conda_py3.5_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.5
- binary_macos_conda:
name: nightly_binary_macos_conda_py3.6
python_version: "3.6"
- binary_conda_upload:
name: nightly_binary_macos_conda_py3.6_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.6
- binary_macos_conda:
name: nightly_binary_macos_conda_py3.7
python_version: "3.7"
- binary_conda_upload:
name: nightly_binary_macos_conda_py3.7_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.7
Loading