diff --git a/.gitignore b/.gitignore index 7ef70e1..bcf6d33 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ pandas-wheels pandas-feedstock pandas dist +.vscode diff --git a/Dockerfile b/Dockerfile index d06874e..55aef52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,3 +6,18 @@ RUN apt-get update && \ RUN conda update conda -y \ && conda install -y conda-build conda-verify gcc_linux-64 gxx_linux-64 \ && conda clean --all + +# ARG USER_ID +# ARG GROUP_ID + +# RUN addgroup --gid $GROUP_ID user +# RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user + +# # https://github.com/ContinuumIO/docker-images/issues/151 +# RUN mkdir /opt/conda/envs/user && \ +# chgrp user /opt/conda/pkgs && \ +# chmod g+w /opt/conda/pkgs && \ +# touch /opt/conda/pkgs/urls.txt && \ +# chown user /opt/conda/envs/user /opt/conda/pkgs/urls.txt + +# USER user diff --git a/Makefile b/Makefile index c3d3cab..b9db32a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # TO EDIT -TAG ?= v1.1.0 +TAG ?= v1.4.3 +GH_USERNAME ?= simonjayhawkins -GH_USERNAME ?= TomAugspurger PANDAS_VERSION=$(TAG:v%=%) PANDAS_BASE_VERSION=$(shell echo $(PANDAS_VERSION) | awk -F '.' '{OFS="."} { print $$1, $$2}') TARGZ=pandas-$(PANDAS_VERSION).tar.gz @@ -14,15 +14,21 @@ SHELL := /bin/bash # ----------------------------------------------------------------------------- init-repos: - git clone https://github.com/pandas-dev/pandas && git -C pandas remote rename origin upstream && git -C pandas remote add origin https://github.com/$(GH_USERNAME)/pandas - git clone https://github.com/conda-forge/pandas-feedstock && git -C pandas-feedstock remote rename origin upstream && git -C pandas-feedstock remote add origin https://github.com/$(GH_USERNAME)/pandas-feedstock - git clone --recursive https://github.com/MacPython/pandas-wheels && git -C pandas-wheels remote rename origin upstream && git -C pandas-wheels remote add origin https://github.com/$(GH_USERNAME)/pandas-wheels + git clone https://github.com/pandas-dev/pandas \ + && git -C pandas remote rename origin upstream \ + && git -C pandas remote add origin https://github.com/$(GH_USERNAME)/pandas + git clone https://github.com/MacPython/pandas-wheels \ + && git -C pandas-wheels remote rename origin upstream \ + && git -C pandas-wheels remote add origin https://github.com/$(GH_USERNAME)/pandas-wheels + git clone https://github.com/conda-forge/pandas-feedstock \ + && git -C pandas-feedstock remote rename origin upstream \ + && git -C pandas-feedstock remote add origin https://github.com/$(GH_USERNAME)/pandas-feedstock + update-repos: - git -C pandas checkout master && git -C pandas pull + git -C pandas checkout main && git -C pandas pull git -C pandas-wheels checkout master && git -C pandas-wheels pull - git -C pandas-feedstock checkout master && git -C pandas-feedstock pull - pushd pandas-wheels && git submodule update --recursive --remote && popd + git -C pandas-feedstock checkout main && git -C pandas-feedstock pull # ----------------------------------------------------------------------------- # Git Tag @@ -37,25 +43,24 @@ tag: # Builder Images # ----------------------------------------------------------------------------- -docker-image: pandas +docker-image: docker build -t pandas-build . - - docker-doc: - docker build -t pandas-docs -f docker-files/docs/Dockerfile . + DOCKER_BUILDKIT=1 docker build --progress=plain -t pandas-docs -f docker-files/docs/Dockerfile . # ----------------------------------------------------------------------------- # sdist # ----------------------------------------------------------------------------- -pandas/dist/$(TARGZ): +sdist: docker run -it --rm \ --name=pandas-sdist-build \ -v ${CURDIR}/pandas:/pandas \ -v ${CURDIR}/scripts:/scripts \ pandas-build \ sh /scripts/build_sdist.sh + sudo chown -R $(shell id -u):$(shell id -g) pandas/dist/ # ----------------------------------------------------------------------------- # Tests @@ -69,7 +74,7 @@ conda-test: -v ${CURDIR}/pandas:/pandas \ -v ${CURDIR}/recipe:/recipe \ pandas-build \ - sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist" + sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist -c conda-forge" pip-test: pandas/dist/$(TARGZ) docker run -it --rm \ @@ -88,7 +93,7 @@ doc: -v ${CURDIR}/pandas:/pandas \ -v ${CURDIR}/scripts/build-docs.sh:/build-docs.sh \ pandas-docs \ - /build-docs.sh + /bin/bash upload-doc: @@ -116,8 +121,9 @@ conda-forge: wheels: - rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp37m-linux_x86_64.whl - rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp37-cp37m-linux_x86_64.whl + rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38m-linux_x86_64.whl + rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38-cp38m-linux_x86_64.whl + rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38-cp38-linux_x86_64.whl ./scripts/wheels.sh $(TAG) $(GH_USERNAME) diff --git a/README.md b/README.md index 0783348..4ac9725 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ -Release automation for pandas. +# Release automation for pandas. -**Windows users should follow the steps in `windows.md`** +**Windows users should follow [these](./windows-wsl.md) steps.** ## Steps to a release -- [ ] Manually update +The `environment.yml` contains the local dependencies. You'll +also need docker. + +And fork pandas-wheels to your GitHub account. + +- [ ] Manually update - [ ] `TAG` in `Makefile` - [ ] `GH_USERNAME` in `Makefile` @@ -14,11 +19,6 @@ If running for the first time be sure to initialize repos make init-repos ``` -The `environment.yml` contains the local dependencies. You'll -also need docker. - -And fork pandas-feedstock and pandas-wheels to your GitHub account. - ``` # Update repos make update-repos @@ -27,52 +27,54 @@ make update-repos make tag # Build the doc and test images -make docker-image docker-doc +make docker-image +make docker-doc # seperate terminal - takes 10-20 mins to solve env -# Build the sdist -make pandas/dist/<>.tar.gz +# Build the sdist (can do this in parallel with building docker-doc image above) +make sdist # Final Pip and Conda tests. Do these in parallel. -# You can optionally do make doc here as well make pip-test make conda-test -# Push the tag. No going back now. -make push-tag -``` +# Docs. You can cheat and re-tag / rebuild these if needed. +make doc +./build-docs.sh +exit -Now manually create a release https://github.com/pandas-dev/pandas/releases +# Visually check docs +python -m http.server --directory pandas/doc/build/html/ +python -m http.server --directory pandas/doc/build/latex/ -Make sure to upload the sdist that's in `pandas/dist/` as the "binary". -Conda-forge uses it. +# Push the tag. No going back now. +make push-tag -On pandas you should also now create and tag a new branch, so +# you may also need to create and tag a new branch, so for example: -```sh -git checkout -b .x -git push upstream .x -git checkout master -git commit --allow-empty -m "Start " -git tag -a v.dev0 -m 'DEV: Start cycle' -git push upstream master --follow-tags +cd pandas +git checkout -b 1.4.x +git push upstream 1.4.x +git checkout main +git commit --allow-empty -m "Start 1.5.0" +git tag -a v1.5.0.dev0 -m 'DEV: Start 1.5.0' +git push upstream main --follow-tags +cd .. ``` -Start the binary builds. **For Mac users** you may need to download the GNU version of sed before running this scripts via `brew install gnu-sed` +Start the binary build. **For Mac users** you may need to download the GNU version of sed before running this scripts via `brew install gnu-sed` ``` -# Binaries -make conda-forge make wheels ``` -Open PRs for each of those. +Open a PR at https://github.com/MacPython/pandas-wheels/pulls. Note that `make wheels` actually pushes a job to MacPython to produce wheels which we will download later. -Docs. You can cheat and re-tag / rebuild these if needed. +While the wheels are building, upload the built docs to the web server ``` -make doc +make upload-doc ``` Once the binaries finish, you'll need to manually upload the wheels to PyPI. @@ -83,6 +85,17 @@ Assuming the job which `make wheels` triggered on MacPython completed successful make download-wheels ``` +if rc0, create version link for the docs to use in the github release announcement + +``` +make link-version +``` + +Now manually create a release https://github.com/pandas-dev/pandas/releases + +Make sure to upload the sdist that's in `pandas/dist/` as the "binary". +Conda-forge uses it. + Upload the wheels and sdist ``` @@ -91,17 +104,22 @@ make upload-pypi Finalize the docs -``` -make upload-doc -``` To make sure /stable and the latest minor revision point to the new release run the following. +**DO NOT DO THIS STEP FOR A RELEASE CANDIDATE** + ``` make link-stable make link-version ``` +if rc we need to manually create a PR since the conda-forge bot does not do this automatically on the dev branch. + +``` +make conda-forge +``` + goto announce. @@ -111,16 +129,6 @@ goto announce. - [ ] Announce Mailing List - [ ] Announce Twitter ------ - -## Initial Setup - -``` -# 1. Initialize Git Repositories -make init-repos - -`````` - --- TODO: diff --git a/docker-files/docs/Dockerfile b/docker-files/docs/Dockerfile index 9646d79..12c7119 100644 --- a/docker-files/docs/Dockerfile +++ b/docker-files/docs/Dockerfile @@ -2,8 +2,11 @@ FROM continuumio/miniconda3:latest RUN apt-get update && apt-get install -y texlive-full xclip \ && apt-get clean + COPY pandas /pandas -RUN conda env create --file=/pandas/environment.yml --name=pandas \ - && . activate pandas \ - && conda uninstall -y --force pandas \ - || true python setup.py build_ext -j 4 + +RUN --mount=type=cache,target=/opt/conda/pkgs conda update conda -n base -y + +RUN --mount=type=cache,target=/opt/conda/pkgs conda install mamba -n base -c conda-forge -y + +RUN --mount=type=cache,target=/opt/conda/pkgs mamba env create --file=/pandas/environment.yml --name=pandas diff --git a/docker-files/windows/Dockerfile b/docker-files/windows/Dockerfile deleted file mode 100644 index 836f159..0000000 --- a/docker-files/windows/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM continuumio/miniconda3:latest - -RUN apt-get update && \ - apt-get install -y make rsync && \ - apt-get clean - -RUN conda update conda -y - -RUN git clone https://github.com/pandas-dev/pandas-release - -RUN conda env update -n base --file=pandas-release/environment.yml - -ARG GH_USERNAME - -ENV GH_USERNAME=$GH_USERNAME - -WORKDIR pandas-release - -RUN make init-repos - -RUN git config --global user.email "pandas-dev@python.org" - -RUN git config --global user.name "Pandas Development Team" diff --git a/recipe/meta.yaml b/recipe/meta.yaml index bec166a..676a1cf 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -18,7 +18,7 @@ requirements: host: - python - pip - - cython >=0.29.13 + - cython >=0.29.30, <3 - numpy run: - python @@ -29,10 +29,11 @@ requirements: test: requires: - pytest + - pytest-xdist - pytest-mock - hypothesis commands: - - python -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db'])" + - python -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db', '-n=2'])" about: home: http://pandas.pydata.org diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 7991753..0d9898c 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -4,17 +4,13 @@ set -e source activate pandas conda uninstall -y --force pandas ||: -# this should have been built by pip-test -# ugh that's 3.7, this is 3.6. boooooo -# actually, can't really do that, since we upload the -# dist folder to PyPI, and we don't want to upload this one... -# python3 -m pip install --no-deps --no-index --find-links=/pandas/dist --only-binary=pandas pandas -apt-get update && apt-get install -y build-essential cd /pandas + python setup.py build_ext -i -j 4 cd /pandas/doc +./make.py clean ./make.py html ./make.py zip_html ./make.py latex_forced ||: diff --git a/scripts/conda-forge.sh b/scripts/conda-forge.sh index 4d5c890..010db5b 100755 --- a/scripts/conda-forge.sh +++ b/scripts/conda-forge.sh @@ -22,7 +22,7 @@ git remote set-url origin https://github.com/${GH_USERNAME}/pandas-feedstock if [[ ${PANDAS_VERSION} == *"rc"* ]]; then git checkout dev else - git checkout master + git checkout main fi git pull upstream diff --git a/scripts/pip_test.sh b/scripts/pip_test.sh index d0555bc..3d75c7b 100755 --- a/scripts/pip_test.sh +++ b/scripts/pip_test.sh @@ -2,10 +2,10 @@ set -e # Cython is *not* required here. -conda create -n pip-test -y python=3.7 numpy pytz python-dateutil pytest pytest-mock hypothesis nomkl +conda create -n pip-test -y python=3.8 numpy pytz python-dateutil pytest pytest-xdist pytest-mock hypothesis nomkl source activate pip-test python3 -m pip wheel --no-deps --wheel-dir=/pandas/dist $1 python3 -m pip install --no-deps --no-index --find-links=/pandas/dist --only-binary=pandas pandas -python3 -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db'])" +python3 -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db', '-n=2'])" diff --git a/scripts/push-tag.py b/scripts/push-tag.py index 89c75b0..c3897e4 100755 --- a/scripts/push-tag.py +++ b/scripts/push-tag.py @@ -20,15 +20,16 @@ def check_tag(tag): def get_branch(tag): """ >>> get_branch("v0.24.0rc0") - 'master' + 'main' >>> get_branch("v0.24.0") - 'master' + '0.24.x' >>> get_branch("v0.24.1") + '0.24.x' """ ver = version.parse(tag.lstrip('v')) - if ver.base_version[-1] == '0': - # off master - base = 'master' + if 'rc0' in tag: + # off main + base = 'main' else: base = '.'.join([tag[1:].rsplit('.', 1)[0], 'x']) diff --git a/scripts/tag.py b/scripts/tag.py index 4820436..d8e7372 100755 --- a/scripts/tag.py +++ b/scripts/tag.py @@ -22,9 +22,9 @@ def check_tag(tag): def checkout(tag): - if tag[-1] == '0' or 'rc' in tag: - # off master - base = 'master' + if 'rc0' in tag: + # off main + base = 'main' else: base = '.'.join([tag[1:].rsplit('.', 1)[0], 'x']) diff --git a/windows-wsl.md b/windows-wsl.md new file mode 100644 index 0000000..16bfb51 --- /dev/null +++ b/windows-wsl.md @@ -0,0 +1,82 @@ +# README for MS Windows Users using WSL 2. + +## Preparing for Your First Release. + +Install Docker + +Ensure the following repositories are forked to your GitHub account + - https://github.com/pandas-dev/pandas-release + - https://github.com/MacPython/pandas-wheels + - https://github.com/pandas-dev/pandas + +see https://docs.microsoft.com/en-us/windows/wsl/install-win10 for Windows Subsystem for Linux installation instructions + + +### Preparing the release environment. + +Start Ubuntu 20.04 LTS + +either configure git manually + +``` +git config --global user.email "" +git config --global user.name "" +git config --global pull.rebase false +``` + +or copy the configuration from windows. +``` +cp /mnt/c/Users//.gitconfig . +``` + +also copy your ssh config from windows and set permissions +``` +cp -r /mnt/c/Users//.ssh/ . +chmod 400 .ssh/id_rsa +``` + +clone your fork of the pandas-release repo and set upstream +``` +git clone git@github.com:/pandas-release.git +cd pandas-release +git remote add upstream https://github.com/pandas-dev/pandas-release.git +``` + +download conda from https://docs.conda.io/en/latest/miniconda.html#linux-installers and +install. +``` +cd +ln -s /mnt/c/Users//Downloads/ ~/downloads +sha256sum downloads/Miniconda3-latest-Linux-x86_64.sh +bash downloads/Miniconda3-latest-Linux-x86_64.sh +``` + +close terminal and reopen to activate conda and install pandas-release conda environment +``` +conda list +conda update conda +cd pandas-release +conda env create -f environment.yml +conda activate pandas-release +``` + +The linux environment is now configured on WSL. Now follow the steps in [README.md](./README.md) + +## View the documentation once built. + +Note: The Linux filesystem is accessible using File Explorer from the Dev Channel of the Windows Insider Program. + +otherwise first find the path to the browser executable, e.g. Microsoft Edge +``` +find /mnt/c/ -name "msedge.exe" 2>/dev/null +``` + +then open index.html in the browser +``` +"/mnt/c/Program Files (x86)/Microsoft/Edge/Application/msedge.exe" pandas/doc/build/html/index.html +``` + +and the same for the pdf documentation +``` +"/mnt/c/Program Files (x86)/Microsoft/Edge/Application/msedge.exe" pandas/doc/build/latex/pandas.pdf +``` diff --git a/windows.md b/windows.md deleted file mode 100644 index 37da179..0000000 --- a/windows.md +++ /dev/null @@ -1,278 +0,0 @@ -# README for MS Windows Users - -## Preparing for Your First Release - -Install Docker - -Ensure the following repositories are forked to your GitHub account - - https://github.com/conda-forge/pandas-feedstock - - https://github.com/MacPython/pandas-wheels - - https://github.com/pandas-dev/pandas - -Open an Anaconda Prompt - - -Build a Docker image with the conda base environment configured and repositories initialized. - -**change GH_USERNAME to your Github username** - -``` -docker build -t pandas-release --build-arg GH_USERNAME=simonjayhawkins -f docker-files/windows/Dockerfile . -``` - -## Preparing the Release Environment for a New Release - -Next we will prepare a Docker container for interactive use during the release process and copy the repositories -from the Docker image into a Docker volume for building the distribution and testing. - -To start with a clean volume. (after a previous release) - -``` -docker volume rm pandas-release -``` - -**change TAG to the release version** -``` -docker run -it --env TAG=v1.0.5 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash -``` - -The Docker release container should be now be running. - -Make sure the repos are up-to-date - -``` -make update-repos -``` - -Tag the release. (This doesn't push the tag) - -``` -make tag -``` - -Stop the container. -``` -exit -``` - -## Preparing the Build and Test Environment - - -Create the Docker image for the sdist build, pip test and conda test containers - -``` -docker build -t pandas-build . -``` - -## Build the sdist - -``` -docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash - -ln -s pandas-release/pandas pandas - -cd pandas-release/ - -./scripts/build_sdist.sh - -exit -``` - -## Pip Tests - - -**change filename to the release version** - -``` -docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash - -ln -s pandas-release/pandas pandas - -cd pandas-release/ - -./scripts/pip_test.sh /pandas/dist/pandas-1.0.5.tar.gz - -exit - -``` - -## Conda Tests - - **change PANDAS_VERSION to the release version** - -``` -docker run -it --rm --env PANDAS_VERSION=1.0.5 -v pandas-release:/pandas-release pandas-build /bin/bash - -ln -s pandas-release/pandas pandas - -cd pandas-release/ - -conda build --numpy=1.17.3 --python=3.8 ./recipe --output-folder=/pandas/dist - -exit - -``` - -## Copy the sdist File from the Docker Volume to the Local Host. - -**change filename to the release version** - -``` -docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp /pandas-release/pandas/dist/pandas-1.0.5.tar.gz /local/" -``` - -## Push the Tag. - -**No going back now.** - -Restart the release container. - -``` -docker start pandas-release -i - -make push-tag - -exit -``` - -## Create New Branch -(not needed for Patch release) - -On pandas you should also now create and tag a new branch, so - -... - -## Create a Release - -Now manually create a release https://github.com/pandas-dev/pandas/releases - -Make sure to upload the sdist as the "binary". Conda-forge uses it. - - -## Start the Binary Builds. - -Restart the release container. - -``` -docker start pandas-release -i - -make conda-forge - -make wheels - -exit -``` - -Open PRs for each of those. - -Note that `make wheels` actually pushes a job to MacPython to produce wheels which we will download later. - - -## Build the Docs. -You can cheat and re-tag / rebuild these if needed. - -``` -docker run -it --name=pandas-docs -v pandas-release:/pandas-release pandas-docs /bin/bash - -rm -r pandas - -ln -s pandas-release/pandas pandas - -cd pandas-release/ - -# following maybe necessary to prevent segfaults -conda update -n base -c defaults conda - -conda env create --file=/pandas/environment.yml --name=pandas - -./scripts/build-docs.sh - -exit -``` - -Copy the built doc files to host and manually inspect html and pdf docs. - -**first remove the local pandas-docs directory (just manually use file manager for now)** - -``` -docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" -``` - -## Upload the Docs - -Copy ssh key and config into release container and restart container. - -``` -docker cp %userprofile%/.ssh pandas-release:/root/.ssh - -docker start pandas-release -i - -chmod 400 ~/.ssh/id_rsa - -make upload-doc - -exit -``` - -## Upload the Binarys to PyPI - -Once the binaries finish, you'll need to manually upload the wheels to PyPI. - -Assuming the job which `make wheels` triggered on MacPython completed successfully (you may want to double check this https://anaconda.org/multibuild-wheels-staging/pandas/files) you can download a copy of the wheels locally. - -``` -docker start pandas-release -i - -make download-wheels - -make upload-pypi - -exit -``` - -## Finalize the Docs - -Do this once the wheels are available on PyPI. - -``` -docker start pandas-release -i - -make link-stable - -make link-version - -exit -``` - -# Announce - -- [ ] Announce Mailing List -- [ ] Announce Twitter