From 5fb7b73f06f46804c81eb1465f388a1ee903fd36 Mon Sep 17 00:00:00 2001 From: shudson Date: Mon, 8 Jul 2019 15:24:31 -0500 Subject: [PATCH] Add release process to documentation --- docs/dev_guide/dev_API/developer_API.rst | 17 +++ .../dev_API/env_resources_module.rst | 10 ++ docs/dev_guide/dev_API/history_module.rst | 10 ++ docs/dev_guide/dev_API/manager_module.rst | 11 ++ .../dev_API/mpi_resources_module.rst | 9 ++ .../dev_API/node_resources_module.rst | 4 + docs/dev_guide/dev_API/resources_module.rst | 16 +++ docs/dev_guide/dev_API/worker_module.rst | 11 ++ .../release_management/release_index.rst | 9 ++ .../release_platforms/index.rst | 10 ++ .../release_platforms/rel_github.rst | 18 +++ .../release_platforms/rel_pypi.rst | 37 +++++ .../release_platforms/rel_spack.rst | 133 ++++++++++++++++++ .../release_management/release_process.rst | 51 +++++++ docs/index.rst | 1 + 15 files changed, 347 insertions(+) create mode 100644 docs/dev_guide/dev_API/developer_API.rst create mode 100644 docs/dev_guide/dev_API/env_resources_module.rst create mode 100644 docs/dev_guide/dev_API/history_module.rst create mode 100644 docs/dev_guide/dev_API/manager_module.rst create mode 100644 docs/dev_guide/dev_API/mpi_resources_module.rst create mode 100644 docs/dev_guide/dev_API/node_resources_module.rst create mode 100644 docs/dev_guide/dev_API/resources_module.rst create mode 100644 docs/dev_guide/dev_API/worker_module.rst create mode 100644 docs/dev_guide/release_management/release_index.rst create mode 100644 docs/dev_guide/release_management/release_platforms/index.rst create mode 100644 docs/dev_guide/release_management/release_platforms/rel_github.rst create mode 100644 docs/dev_guide/release_management/release_platforms/rel_pypi.rst create mode 100644 docs/dev_guide/release_management/release_platforms/rel_spack.rst create mode 100644 docs/dev_guide/release_management/release_process.rst diff --git a/docs/dev_guide/dev_API/developer_API.rst b/docs/dev_guide/dev_API/developer_API.rst new file mode 100644 index 0000000000..be0798b55a --- /dev/null +++ b/docs/dev_guide/dev_API/developer_API.rst @@ -0,0 +1,17 @@ +libEnsemble Internal Modules +============================ + +Internal modules of libEnsemble. + + +.. toctree:: + :maxdepth: 1 + :caption: Modules: + + manager_module + worker_module + history_module + resources_module + mpi_resources_module + env_resources_module + node_resources_module diff --git a/docs/dev_guide/dev_API/env_resources_module.rst b/docs/dev_guide/dev_API/env_resources_module.rst new file mode 100644 index 0000000000..08747268ab --- /dev/null +++ b/docs/dev_guide/dev_API/env_resources_module.rst @@ -0,0 +1,10 @@ +Environment Resources Module +============================ +.. automodule:: env_resources +.. :members: EnvResources + +.. autoclass:: EnvResources + :member-order: bysource + :members: + + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/history_module.rst b/docs/dev_guide/dev_API/history_module.rst new file mode 100644 index 0000000000..57c97d821c --- /dev/null +++ b/docs/dev_guide/dev_API/history_module.rst @@ -0,0 +1,10 @@ +History Module +============== +.. automodule:: history +.. :members: History + +.. autoclass:: History + :member-order: bysource + :members: + + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/manager_module.rst b/docs/dev_guide/dev_API/manager_module.rst new file mode 100644 index 0000000000..c0917af211 --- /dev/null +++ b/docs/dev_guide/dev_API/manager_module.rst @@ -0,0 +1,11 @@ +Manager Module +============== +.. automodule:: libE_manager + :members: manager_main + :undoc-members: + +.. autoclass:: Manager + :member-order: bysource + :members: + + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/mpi_resources_module.rst b/docs/dev_guide/dev_API/mpi_resources_module.rst new file mode 100644 index 0000000000..6da0e66756 --- /dev/null +++ b/docs/dev_guide/dev_API/mpi_resources_module.rst @@ -0,0 +1,9 @@ +MPI Resources Module +==================== +.. automodule:: mpi_resources +.. :members: MPIResources + +.. autoclass:: MPIResources + :show-inheritance: + :member-order: bysource + :members: diff --git a/docs/dev_guide/dev_API/node_resources_module.rst b/docs/dev_guide/dev_API/node_resources_module.rst new file mode 100644 index 0000000000..71afcb0320 --- /dev/null +++ b/docs/dev_guide/dev_API/node_resources_module.rst @@ -0,0 +1,4 @@ +Node Resources Module +===================== +.. automodule:: node_resources + :members: diff --git a/docs/dev_guide/dev_API/resources_module.rst b/docs/dev_guide/dev_API/resources_module.rst new file mode 100644 index 0000000000..cc998cbf9c --- /dev/null +++ b/docs/dev_guide/dev_API/resources_module.rst @@ -0,0 +1,16 @@ +Resources Module +================ +.. automodule:: resources +.. :members: Resources + +.. autoclass:: Resources + :member-order: bysource + :members: + + .. automethod:: __init__ + +.. autoclass:: WorkerResources + :member-order: bysource + :members: + + .. automethod:: __init__ diff --git a/docs/dev_guide/dev_API/worker_module.rst b/docs/dev_guide/dev_API/worker_module.rst new file mode 100644 index 0000000000..855e39c4d5 --- /dev/null +++ b/docs/dev_guide/dev_API/worker_module.rst @@ -0,0 +1,11 @@ +Worker Module +============= +.. automodule:: libE_worker + :members: worker_main + +.. autoclass:: Worker + :member-order: bysource + :members: + + .. automethod:: __init__ + diff --git a/docs/dev_guide/release_management/release_index.rst b/docs/dev_guide/release_management/release_index.rst new file mode 100644 index 0000000000..27102ba253 --- /dev/null +++ b/docs/dev_guide/release_management/release_index.rst @@ -0,0 +1,9 @@ +Release Management for libEnsemble +================================== + +.. toctree:: + :maxdepth: 2 + :caption: libEnsemble Release Contents: + + release_process.rst + release_platforms/index.rst diff --git a/docs/dev_guide/release_management/release_platforms/index.rst b/docs/dev_guide/release_management/release_platforms/index.rst new file mode 100644 index 0000000000..f2ededcfad --- /dev/null +++ b/docs/dev_guide/release_management/release_platforms/index.rst @@ -0,0 +1,10 @@ +Release Platforms +================= + +.. toctree:: + :maxdepth: 1 + :caption: libEnsemble Release Platforms: + + rel_github.rst + rel_pypi.rst + Spack release diff --git a/docs/dev_guide/release_management/release_platforms/rel_github.rst b/docs/dev_guide/release_management/release_platforms/rel_github.rst new file mode 100644 index 0000000000..8e59107f84 --- /dev/null +++ b/docs/dev_guide/release_management/release_platforms/rel_github.rst @@ -0,0 +1,18 @@ +.. _rel-github: + +Github release +============== + +The administrator should follow the github instructions to draft a new release. These can currently be found at: https://help.github.com/en/articles/creating-releases + +Both the version and title will be of the form vX.Y.Z:: + + E.g. v0.5.0. + +From version 1.0, these should follow semantic versioning where, where X/Y/Z are major, minor and patch revisions. + +Prior to version 1.0, the second number may include breaking API changes, and the third number may include minor additions. + +The release notes should be included in the description. These should already be in `docs/release_notes.rst`. The release nots should be copied just the current release, starting from the date. Hint: To see example of raw input click *edit* next to one of the previous releases. + +Note, unlike some platforms (e.g. PyPI), github releases can be edited or deleted once created. diff --git a/docs/dev_guide/release_management/release_platforms/rel_pypi.rst b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst new file mode 100644 index 0000000000..906d250af9 --- /dev/null +++ b/docs/dev_guide/release_management/release_platforms/rel_pypi.rst @@ -0,0 +1,37 @@ +.. _rel-pypi: + +PyPI release +============ + +libEnsemble is released on the Python Package Index (commonly known as PyPI). This enable users to “pip install” the package. + +The package is stored on PyPI in the form of a source distribution (commonly known as a tarball). The tarball could be obtained from github, though historically this has been created with a checkout of libensemble from git. + + +You will need logon credentials for the libEnsemble PyPI. You will also need twine (which can be pip or conda installed). + + +In the package directory on the master branch (the one containing setup.py) do the following: + + +Create distribution: + + python setup.py sdist + + +Upload (you will need username/password here): + + twine upload dist/* + + +If you now do "pip install libensemble" it should find the new version. + + +It should also be visible here: + +https://pypi.org/project/libensemble/ + +For more details on creating PyPI packages see https://betterscientificsoftware.github.io/python-for-hpc/tutorials/python-pypi-packaging/ + + + diff --git a/docs/dev_guide/release_management/release_platforms/rel_spack.rst b/docs/dev_guide/release_management/release_platforms/rel_spack.rst new file mode 100644 index 0000000000..c1b20b4360 --- /dev/null +++ b/docs/dev_guide/release_management/release_platforms/rel_spack.rst @@ -0,0 +1,133 @@ +.. _rel-spack: + +A workflow for updating libEnsemble on Spack +============================================ + +Introduction +------------ + +This assumes you have already: + + - Made a PyPI package for new version of libensemble + - Made a github fork of Spack and cloned it to your local system. + +Details on how to create forks can be found at: https://help.github.com/articles/fork-a-repo + +You now have a configuration like shown in answer at: https://stackoverflow.com/questions/6286571/are-git-forks-actually-git-clones + +Upstream, in this case, is the official Spack repository on github. Origin is your fork on github and Local Machine is your local clone (from your fork). + + +Make sure SPACK_ROOT is set and spack binary is in your path:: + + export SPACK_ROOT= + export PATH=$SPACK_ROOT/bin:$PATH + + +DO ONCE in your local checkout: + +To set upstream repo:: + + git remote add upstream https://github.com/spack/spack.git + git remote -v # check added + +(Optional) Prevent accidental pushes to upstream:: + + git remote set-url --push upstream no_push + git remote -v # Check for line: `upstream no_push (push)` + + +Now to update (the main develop branch) +--------------------------------------- + +You will now update your local machine from the upstream repo (if in doubt - make a copy of local repo +in your filestystem before doing the following). + +Check upstream remote is present:: + + git remote -v + +Ensure you are on the develop branch:: + + git checkout develop + +Fetch from upstream repo:: + + git fetch upstream + +Now to update your local machine you may wish to rebase or overwrite your local files. +Select from the following: + +If you have local changes to go "on top" of latest code:: + + git rebase upstream/develop + +Or to make your local machine identical to upstream repo (**WARNING** Any local changes WILL BE LOST):: + + git reset --hard upstream/develop + + +(Optional) You may want to update your forked (origin) repo on github at this point. +This may requires a forced push:: + + git push origin develop --force + + +Making changes +-------------- + +You can optionally create a branch to make changes on. This may be a good idea, especially if +you have multiple packages, to make separate branches for each package. + +See the Spack [packaging](https://spack.readthedocs.io/en/latest/packaging_guide.html) and +[contibution](https://spack.readthedocs.io/en/latest/contribution_guide.html) guides for more info. + + +Quick example to update libensemble:: + + git branch update_libensemble + git checkout update_libensemble + +This will open the libensemble package.py file in your editor (given by env variable EDITOR):: + + spack edit py-libensemble # SPACK_ROOT must be set (see above) (python packages use "py-" prefix) + +Or just open it manually: var/spack/repos/builtin/packages/py-libensemble/package.py + + +Now get checksum for new lines: + +Get the tarball (see PyPI instructions), for the new release and use:: + + sha256sum libensemble-*.tar.gz + +Update the `package.py` file by pasting in the new checksum lines (and make sure url line points to latest version). +Also update any dependencies for the new version. + +Check package:: + + spack flake8 + +If OK add, commit and push to origin (forked repo):: + + git commit -am "Update libensemble" + git push origin update_libensemble --force + +Once the branch is pushed to the forked repo - go to github and do a pull request from this +branch on the fork to the develop branch on the upstream. + + +Express summary: Make fork identical to upstream +------------------------------------------------ + +Quick summary for bringing develop branch on forked repo up to speed with upstream +(YOU WILL LOSE ANY CHANGES):: + + git remote add upstream https://github.com/spack/spack.git + git fetch upstream + git checkout develop + git reset --hard upstream/develop + git push origin develop --force + +Reference: + diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst new file mode 100644 index 0000000000..99e55c41c7 --- /dev/null +++ b/docs/dev_guide/release_management/release_process.rst @@ -0,0 +1,51 @@ +Release Process +=============== + +This document details the current release process for libEnsemble. A release can only be undertaken by a project administrator. A project administrator should have an administrator role on the libEnsemble github, PyPI and readthedocs pages. + +Before release +-------------- + +- A github issue is created with a checklist for the release. + +- A release branch should be taken off develop (or develop pulls controlled). + +- Release notes for this version are added to the documentation with release date, including a list of supported (tested) platforms. + +- Version number is updated wherever it appears (in `setup.py`, `libensemble/__init__.py` and `docs/conf.py`) + +- setup.py and `libensemble/__init__.py` are checked to ensure all information is up to date. + +- Tests are run with source to be released (this may iterate): + + - On-line CI (currently Travis) tests must pass. + + - Scaling tests must be run on HPC platforms listed as supported in release notes. + + - Coverage must not have decreased unless there is a justifiable reason. + + - Documentation must build and display correctly wherever hosted (currently readthedocs.com). + + +- Pull request from either develop or release branch to master requesting reviewer/s (including at least one other administrator). + +- Reviewer will check tests have passed and approve merge. + + +During release +-------------- + +An administrator will take the following steps. + +- Merge the pull request into master. + +- Once CI tests have passed on master. + + - A github release will be taken from the master (:ref:`github release`). + + - A tarball (source distribution) will be uploaded to PyPI (:ref:`PyPI release`). + + - Spack package will be updated (:ref:`Spack release`). + + +- If the merge was made from a release branch (instead of develop), merge this branch into develop. diff --git a/docs/index.rst b/docs/index.rst index 0a6c406839..a9d01454ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,6 +68,7 @@ libEnsemble is a library for managing ensemble-like collections of computations. :maxdepth: 2 :caption: Developer Guide: + dev_guide/release_management/release_index.rst dev_guide/dev_API/developer_API.rst