diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index 3982fce7314..00000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# CircleCI Script Usage - -## Tags/Releases build automatically - -If you just push a tag, circleci will build that tag and make its artifacts available both on CircleCI and by creating a GitHub release draft containing the artifacts. This is a result of the tag_build *workflow*, which calls the job. diff --git a/.circleci/aur-checker-Dockerfile b/.circleci/aur-checker-Dockerfile new file mode 100644 index 00000000000..c6f4fb94302 --- /dev/null +++ b/.circleci/aur-checker-Dockerfile @@ -0,0 +1,11 @@ +# This can be built and pushed with +# cat aur-checker-Dockerfile | docker build -t "drud/arch-aur-builder:latest" - && docker push "drud/arch-aur-builder:latest" +# - Edit PKGBUILD to change the version and hash or anything else +# - Then run it with +# docker run --rm --mount type=bind,source=$(pwd),target=/tmp/ddev-bin --workdir=/tmp/ddev-bin drud/arch-aur-builder bash -c "makepkg --printsrcinfo > .SRCINFO && makepkg -s" +# Then `git add -u` and commit and push +FROM archlinux:latest +RUN echo | pacman -Sy sudo binutils fakeroot docker docker-compose +RUN useradd builder +USER builder +CMD bash diff --git a/.circleci/bump_aur.sh b/.circleci/bump_aur.sh new file mode 100755 index 00000000000..453bf9119de --- /dev/null +++ b/.circleci/bump_aur.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Requires $AUR_SSH_PRIVATE_KEY, a private key in environment variable +# This environment variable must be a single line, with \n replaced by "" + +set -o errexit +set -o pipefail +set -o nounset + +if [ -z "${AUR_SSH_PRIVATE_KEY:-}" ]; then + printf "\$AUR_SSH_PRIVATE_KEY must be set in the environment. It should be a single line with \n replaced by " && exit 101 +fi +if [ "$#" != "3" ]; then + printf "Arguments: AUR_REPO (AUR repo ddev-bin or ddev-edge-bin) \nVERSION_NUMBER (like v1.14.2) \nARTIFACTS_DIR (like /home/circleci/artifacts)\n" && exit 102 +fi + +# For testing, you can change GITHUB_USERNAME to, for example, rfay so releases can be tested +# without bothering people. +GITHUB_USERNAME=drud +AUR_USERNAME=ddev-releaser +AUR_REPO=$1 +VERSION_NUMBER=$2 +ARTIFACTS_DIR=$3 +NO_V_VERSION=$(echo ${VERSION_NUMBER} | awk -F"-" '{ OFS="-"; sub(/^./, "", $1); printf $0; }') +LINUX_HASH=$(cat $ARTIFACTS_DIR/ddev_linux.${VERSION_NUMBER}.tar.gz.sha256.txt | awk '{print $1;}' ) +LINUX_TARBALL_URL=https://github.com/${GITHUB_USERNAME}/ddev/releases/download/${VERSION_NUMBER}/ddev_linux.${VERSION_NUMBER}.tar.gz +if [ ! -z "${LINUX_TARBALL_OVERRIDE:-}" ]; then + LINUX_TARBALL_URL=${LINUX_TARBALL_OVERRIDE} + LINUX_HASH=$(curl -sSL "${LINUX_TARBALL_URL}.sha256.txt" | awk '{print $1}') +fi + +EDGE_DESCRIPTION="" +if [ ${AUR_REPO} = "ddev-edge-bin" ] ; then EDGE_DESCRIPTION=" (edge channel)"; fi + +# Add temporary private key provided by CircleCI context +echo $AUR_SSH_PRIVATE_KEY | perl -p -e 's//\n/g' >/tmp/id_rsa_aur && chmod 600 /tmp/id_rsa_aur + +ssh-add /tmp/id_rsa_aur +rm -rf ${AUR_REPO} +git config --global core.sshCommand 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' +git clone ssh://aur@aur.archlinux.org/${AUR_REPO}.git && pushd ${AUR_REPO} && touch .BUILDINFO && chmod -R ugo+w . + +_name="ddev" +cat >PKGBUILD < .SRCINFO && makepkg -s" + +git config user.email "randy+ddev-releaser@randyfay.com" +git config user.name "ddev-releaser" + +git commit -am "AUR bump to ${VERSION_NUMBER}" + +git push +popd diff --git a/.circleci/bump_homebrew.sh b/.circleci/bump_homebrew.sh new file mode 100755 index 00000000000..60f1c2cf381 --- /dev/null +++ b/.circleci/bump_homebrew.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Requires $DDEV_GITHUB_TOKEN, a token with public repo power + +set -o errexit +set -o pipefail +set -o nounset + +if [ $# != 4 ]; then + printf "Arguments: GITHUB_REPO (homebrew repo like drud/homebrew-ddev) \nPROJECT_NAME (like ddev) \nVERSION_NUMBER (like v1.14.2) \nARTIFACTS_DIR (like /home/circleci/artifacts)\n" && exit 101 +fi + +# For testing, you can change GITHUB_USERNAME to, for example, rfay so releases can be tested +# without bothering people. +GITHUB_USERNAME=drud +GITHUB_REPO=$1 +PROJECT_NAME=$2 +VERSION_NUMBER=$3 +ARTIFACTS_DIR=$4 +NO_V_VERSION=$(echo ${VERSION_NUMBER} | awk -F"-" '{ OFS="-"; sub(/^./, "", $1); printf $0; }') +SOURCE_URL="https://github.com/drud/ddev/archive/${VERSION_NUMBER}.tar.gz" +SOURCE_SHA=$(curl -sSL ${SOURCE_URL} | shasum -a 256 | awk '{print $1}') + +LINUX_BOTTLE_SHA=$(awk '{print $1}' <"${ARTIFACTS_DIR}/${PROJECT_NAME}-${NO_V_VERSION}.x86_64_linux.bottle.tar.gz.sha256.txt") +MACOS_BOTTLE_SHA=$(awk '{print $1}' <${ARTIFACTS_DIR}/${PROJECT_NAME}-${NO_V_VERSION}.sierra.bottle.tar.gz.sha256.txt) + + +TMPDIR=$(mktemp -d) +cd ${TMPDIR} && git clone https://github.com/${GITHUB_REPO} && cd $(basename ${GITHUB_REPO}) + +cat >Formula/${PROJECT_NAME}.rb < :run + # depends_on "docker-compose" => :run + depends_on "docker" => :build + depends_on "go" => :build + depends_on "mkcert" => :run + depends_on "nss" => :run + + bottle do + root_url "https://github.com/drud/ddev/releases/download/${VERSION_NUMBER}/" + cellar :any_skip_relocation + sha256 "${LINUX_BOTTLE_SHA}" => :x86_64_linux + sha256 "${MACOS_BOTTLE_SHA}" => :sierra + end + def install + system "make", "VERSION=v#{version}", "COMMIT=v#{version}" + system "mkdir", "-p", "#{bin}" + if OS.mac? + system "cp", ".gotmp/bin/darwin_amd64/ddev", "#{bin}/ddev" + system ".gotmp/bin/darwin_amd64/ddev_gen_autocomplete" + else + system "cp", ".gotmp/bin/ddev", "#{bin}/ddev" + system ".gotmp/bin/ddev_gen_autocomplete" + end + bash_completion.install ".gotmp/bin/ddev_bash_completion.sh" => "ddev" + zsh_completion.install ".gotmp/bin/ddev_zsh_completion.sh" => "ddev" + end + + test do + system "#{bin}/ddev", "--version" + end + + def caveats + <<~EOS +Make sure to do a 'mkcert -install' if you haven't done it before, it may require your sudo password. + +ddev requires docker and docker-compose. +Docker installation instructions at https://ddev.readthedocs.io/en/stable/users/docker_installation/ + EOS + end +end +END + +git config user.email "randy@randyfay.com" +git config user.name "rfay" +git add -u +git commit -m "Homebrew bump to ${VERSION_NUMBER}" + +git push -q https://${GITHUB_USERNAME}:${DDEV_GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git master diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f655990755..f15198be2db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -357,45 +357,7 @@ jobs: path: ~/artifacts name: Artifact storage - # 'tag_build' automatically builds a tag . -# tag_build: -# machine: -# image: ubuntu-1604:201903-01 -# working_directory: ~/ddev -# environment: -# DDEV_DEBUG: "true" -# steps: -# - checkout -# - run: sudo mkdir /home/linuxbrew && sudo chown $(id -u) /home/linuxbrew -# - restore_cache: -# keys: -# - linux-v12 -# - run: -# command: ./.circleci/linux_circle_vm_setup.sh -# name: TAG BUILD Circle VM setup - tools, docker, golang -# - save_cache: -# key: linux-v12 -# paths: -# - /home/linuxbrew -# - ~/.ddev/testcache -# -# # Now build using the regular ddev-only technique - this results in a fully clean set of executables. -# - run: -# command: make -s clean linux darwin windows_install EXTRA_PATH=/home/linuxbrew/.linuxbrew/bin -# name: Build the ddev executables -# -# # We only build the xz version of the docker images on tag build. -# - run: -# # Do not build the docker tarballs at simple tag build time -# command: ./.circleci/generate_artifacts.sh ~/artifacts false false -# name: tar/zip up artifacts and make hashes -# no_output_timeout: "40m" -# -# - store_artifacts: -# path: ~/artifacts -# name: Artifact storage - # 'tag_build' is used to push a full release release_build: macos: xcode: "11.3.1" @@ -406,8 +368,8 @@ jobs: - checkout - run: command: ./.circleci/macos_circle_vm_setup.sh - name: TAG BUILD (macOS) Circle VM setup - + name: RELEASE BUILD (macOS) Circle VM setup + - run: echo "version=$(make version) CIRCLE_TAG=${CIRCLE_TAG}" - run: command: make -s clean linux windows_install chocolatey @@ -424,16 +386,14 @@ jobs: - run: command: | if [ ! -z "${DDEV_GITHUB_TOKEN}" ]; then - version=$(make version) - version="${version#*:}" # Remove the VERSION: in front of make version - echo "DDEV_GITHUB_TOKEN provided, pushing release ${version}" + echo "DDEV_GITHUB_TOKEN provided, pushing release ${CIRCLE_TAG}" ghr \ -prerelease \ -r $CIRCLE_PROJECT_REPONAME \ -u $CIRCLE_PROJECT_USERNAME \ -b "$(cat ./.github/RELEASE_NOTES_TEMPLATE.md)" \ -t $DDEV_GITHUB_TOKEN \ - "${version}" ~/artifacts + "${CIRCLE_TAG}" ~/artifacts else echo "DDEV_GITHUB_TOKEN not provided, not pushing release $CIRCLE_TAG" fi @@ -441,20 +401,44 @@ jobs: - store_artifacts: path: ~/artifacts name: Artifact storage - # When fixed, this will have to be done after push to github, so it can use - # the real github release artifact. -# - run: -# name: Upload chocolatey windows release -# command: | -# if [ ! -z "${CHOCOLATEY_API_KEY:-}" ]; then -# echo "Pushing release to chocolatey..." -# pushd .gotmp/bin/windows_amd64/chocolatey -# docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key "${CHOCOLATEY_API_KEY}" -# popd -# else -# echo "NOT pushing release to chocolatey because no CHOCOLATEY_API_KEY was provided" -# fi - + - run: + name: "Bump homebrew edge release" + command: ".circleci/bump_homebrew.sh ${HOMEBREW_EDGE_REPOSITORY} ddev ${CIRCLE_TAG} ~/artifacts" + - run: + name: "Bump homebrew main release if necessary" + command: | + if [ ${CIRCLE_TAG%-*} = ${CIRCLE_TAG} ]; then + .circleci/bump_homebrew.sh ${HOMEBREW_STABLE_REPOSITORY} ddev ${CIRCLE_TAG} ~/artifacts + else + echo "Skipping homebrew main release because ${CIRCLE_TAG} is an edge/prerelease" + fi + - run: + name: Push AUR ddev-bin if necessary + command: | + if [ ! -z "${AUR_SSH_PRIVATE_KEY}" ] && [ "${CIRCLE_TAG%-*}" = "${CIRCLE_TAG}" ] ; then + .circleci/bump_aur.sh ddev-bin ${CIRCLE_TAG} ~/artifacts + else + echo "Skipping AUR ddev-bin push" + fi + - run: + name: Push AUR ddev-edge-bin + command: | + if [ ! -z "${AUR_SSH_PRIVATE_KEY}" ] ; then + .circleci/bump_aur.sh ddev-edge-bin ${CIRCLE_TAG} ~/artifacts + else + echo "Skipping AUR ddev-edge-bin push" + fi + - run: + name: Upload chocolatey windows release + command: | + if [ ! -z "${CHOCOLATEY_API_KEY:-}" ]; then + echo "Pushing release to chocolatey..." + pushd .gotmp/bin/windows_amd64/chocolatey + docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key "${CHOCOLATEY_API_KEY}" + popd + else + echo "NOT pushing release to chocolatey because no CHOCOLATEY_API_KEY was provided" + fi workflows: version: 2 diff --git a/.circleci/linux_circle_vm_setup.sh b/.circleci/linux_circle_vm_setup.sh index e88013be5d5..eaf1122f948 100755 --- a/.circleci/linux_circle_vm_setup.sh +++ b/.circleci/linux_circle_vm_setup.sh @@ -58,7 +58,7 @@ GOTESTSUM_VERSION=0.3.2 curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v$GOTESTSUM_VERSION/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" | sudo tar -xz -C /usr/local/bin gotestsum && sudo chmod +x /usr/local/bin/gotestsum # Install ghr -GHR_RELEASE="v0.12.0" +GHR_RELEASE="v0.13.0" curl -fsL -o /tmp/ghr.tar.gz https://github.com/tcnksm/ghr/releases/download/${GHR_RELEASE}/ghr_${GHR_RELEASE}_linux_amd64.tar.gz sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/ghr.tar.gz ghr_${GHR_RELEASE}_linux_amd64/ghr ghr -v diff --git a/docs/developers/release-checklist.md b/docs/developers/release-checklist.md index fdef1ab6159..7a697bcf93a 100644 --- a/docs/developers/release-checklist.md +++ b/docs/developers/release-checklist.md @@ -1,17 +1,47 @@ # DDEV-Local Release Checklist -1. Create provisional tagged images. `git fetch upstream && git checkout upstream/master && cd containers` and `for item in *; do pushd $item; make push VERSION= DOCKER_ARGS=--no-cache ; popd; done` +## CircleCI Environment Preparation + +The following environment variables must be added to the **org context** named [ddev-local in CircleCI](https://app.circleci.com/settings/organization/github/drud/contexts/fca74bf5-b028-49cf-a42e-ce47fdb79866) + +* AUR_SSH_PRIVATE_KEY: The private ssh key for the ddev-releaser user. This must be processed into a single line, for example, `perl -p -e 's/\n//' ~/.ssh/id_rsa_ddev_releaser| pbcopy`. + +* CHOCOLATEY_API_KEY: API key for chocolatey. + +* DDEV_GITHUB_TOKEN: The github token that gives access to create releases and push to the homebrew repositories. + +* DDEV_MACOS_APP_PASSWORD: The password used for notarization, see [signing_tools](https://github.com/drud/signing_tools) + +* DDEV_MACOS_SIGNING_PASSWORD: The password the access the signing key on macOS, see [signing_tools](https://github.com/drud/signing_tools) + +* DDEV_WINDOWS_SIGNING_PASSWORD: The windows signing password. + +* HOMEBREW_EDGE_REPOSITORY: The name of the github repo used for the edge channel on homebrew, drud/homebrew-ddev-ege + +* HOMEBREW_STABLE_REPOSITORY: The name of the github repo used for the stable channel on homebrew/ drud/homebrew-ddev +* SegmentKey: The key that enabled the Segment reporting + +## Creating a release (almost everything should be automated) + +1. Create provisional tagged images. `git fetch upstream && git checkout upstream/master && cd containers` and `for item in *; do pushd $item; make push VERSION= DOCKER_ARGS=--no-cache ; popd; done` 2. Update the default container versions in `pkg/version/version.go` and create a pull request -3. Ensure all updates have been merged into the master branch -4. Create a release for the new version using the github UI. It should be "prerelease" if it's only an edge release. -5. Add the commit list (`git log vXXX..vYYY --oneline --decorate=no`) to the release page -6. Update the `ddev` homebrew formulas (ddev-edge and ddev) as necessary, and with the source .tar.gz and SHA checksum of the tarball and the bottle builds and tarballs. The bottles and checksums for macOS (sierra) and x86_64_linux are built and pushed to the release page automatically by the CircleCI release build process. Test `brew upgrade ddev` both on macOS and Linux and make sure ddev is the right version and behaves well. -7. Download the ddev_chocolatey tarball and extract it. It should be available on a Windows machine or VM. (It can be network-mounted into a Windows VM). cd into the extraction directory and push it to chocolatey with `choco push -s https://push.chocolatey.org/ --api-key=choco-apikey-a720-asome-api-key` This can also be done from macOS or Linux using mono-choco, `docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key=`. After the push responses have come back, install with `choco install -y --pre ddev --version ` and verify correct behavior. -8. Publish the release to the ddev [AUR repository](https://aur.archlinux.org/packages/ddev-bin/). The README.md in the AUR git repo (`ssh://aur@aur.archlinux.org/ddev-bin.git` or `https://aur.archlinux.org/ddev-bin.git`) has instructions on how to update, including how to do it with a Docker container, so it doesn't have to be done on an ArchLinux or Manjaro VM. -9. Update the release page with full details about the current release -10. Publish the release (unmark it as "prerelease") if it's a normal (non-edge) release -11. On [ReadTheDocs](https://readthedocs.org/projects/ddev/builds) click the button to "build version" "latest". Then on [versions](https://readthedocs.org/projects/ddev/versions/) page make sure that "stable" reflects the hash of the new version. +3. Create a release for the new version using the github UI. It should be "prerelease" if it's only an edge release. +4. Add the commit list (`git log vXXX..vYYY --oneline --decorate=no`) to the release page. +5. Update the release page with full details about the current release +6. Verify that homebrew (linux and macOS) and Chocolatey and AUR are working correctly with the right versions + +## Manually updating homebrew formulas + +* Update the `ddev` homebrew formulas (ddev-edge and ddev) as necessary, and with the source .tar.gz and SHA checksum of the tarball and the bottle builds and tarballs. The bottles and checksums for macOS (sierra) and x86_64_linux are built and pushed to the release page automatically by the CircleCI release build process. Test `brew upgrade ddev` both on macOS and Linux and make sure ddev is the right version and behaves well. + +## Manually updating Chocolatey + +* Download the ddev_chocolatey tarball and extract it. It should be available on a Windows machine or VM. (It can be network-mounted into a Windows VM). cd into the extraction directory and push it to chocolatey with `choco push -s https://push.chocolatey.org/ --api-key=choco-apikey-a720-asome-api-key` This can also be done from macOS or Linux using mono-choco, `docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key=`. After the push responses have come back, install with `choco install -y --pre ddev --version ` and verify correct behavior. + +## Manually updating AUR repository + +* Publish the release to the ddev [AUR repository](https://aur.archlinux.org/packages/ddev-bin/). The README.md in the AUR git repo (`ssh://aur@aur.archlinux.org/ddev-bin.git` or `https://aur.archlinux.org/ddev-bin.git`) has instructions on how to update, including how to do it with a Docker container, so it doesn't have to be done on an ArchLinux or Manjaro VM. ## Manually Signing with Windows installer