Notarize ddev for macOS Catalina (#2015)

.circleci/config.yml

@@ -73,14 +73,14 @@ jobs:
         at: ~/
     - restore_cache:
         keys:
-        - homebrew-macos-v13
+        - homebrew-macos-v14
     # Run the built-in ddev tests with the executables just built.
     - run:
         command: ./.circleci/macos_circle_vm_setup.sh
         name: macOS Circle VM setup - tools, docker, golang
         # Now build using the regular ddev-only technique - this results in a fully clean set of executables.
     - save_cache:
-        key: homebrew-macos-v13
+        key: homebrew-macos-v14
         paths:
         - /usr/local/Homebrew
         - /usr/local/Cellar
@@ -105,7 +105,7 @@ jobs:
         at: ~/
     - restore_cache:
         keys:
-        - homebrew-macos-v13
+        - homebrew-macos-v14
     # Run the built-in ddev tests with the executables just built.
     - run:
         command: ./.circleci/macos_circle_vm_setup.sh
@@ -118,15 +118,15 @@ jobs:
     - store_test_results:
         path: /tmp/testresults
     - save_cache:
-        key: homebrew-macos-v13
+        key: homebrew-macos-v14
         paths:
         - /usr/local/Homebrew
         - /usr/local/Cellar
         - ~/Library/Caches/Homebrew
 
   mac_nfsmount_test:
     macos:
-      xcode: "11.0.0"
+      xcode: "11.3.0"
     working_directory: ~/ddev
     environment:
       DDEV_TEST_USE_NFSMOUNT: "true"
@@ -137,13 +137,13 @@ jobs:
         at: ~/
     - restore_cache:
         keys:
-        - homebrew-macos-v13
+        - homebrew-macos-v14
     # Run the built-in ddev tests with the executables just built.
     - run:
         command: ./.circleci/macos_circle_vm_setup.sh
         name: macOS Circle VM setup - tools, docker, golang
     - save_cache:
-        key: homebrew-macos-v13
+        key: homebrew-macos-v14
         paths:
         - /usr/local/Homebrew
         - /usr/local/Cellar
@@ -255,7 +255,6 @@ jobs:
       image: ubuntu-1604:201903-01
     working_directory: ~/ddev
     environment:
-      ARTIFACTS: /artifacts
     steps:
     - checkout
     - run: sudo mkdir /home/linuxbrew && sudo chown $(id -u) /home/linuxbrew
@@ -305,18 +304,18 @@ jobs:
 
   mac_container_test:
     macos:
-      xcode: "11.0.0"
+      xcode: "11.3.0"
     working_directory: ~/ddev
     steps:
     - checkout
     - restore_cache:
         keys:
-        - homebrew-macos-v13
+        - homebrew-macos-v14
     - run:
         command: ./.circleci/macos_circle_vm_setup.sh
         name: macOS Circle VM setup - tools, docker, golang
     - save_cache:
-        key: homebrew-macos-v13
+        key: homebrew-macos-v14
         paths:
         - /usr/local/Homebrew
         - /usr/local/Cellar
@@ -337,8 +336,6 @@ jobs:
     machine:
       image: ubuntu-1604:201903-01
     working_directory: ~/ddev
-    environment:
-      ARTIFACTS: /artifacts
     steps:
     - run: sudo mkdir /home/linuxbrew && sudo chown $(id -u) /home/linuxbrew
     - restore_cache:
@@ -347,15 +344,15 @@ jobs:
     - attach_workspace:
         at: ~/
     - run:
-        command: ./.circleci/generate_artifacts.sh $ARTIFACTS ${BUILD_IMAGE_TARBALLS:false}
+        command: ./.circleci/generate_artifacts.sh ~/artifacts ${BUILD_IMAGE_TARBALLS:false}
         name: tar/zip up artifacts and make hashes
         no_output_timeout: "40m"
     - save_cache:
         key: homebrew-linux-v9
         paths:
         - /home/linuxbrew
     - store_artifacts:
-        path: /artifacts
+        path: ~/artifacts
         name: Artifact storage
 
   # 'tag_build' automatically builds a tag .
@@ -365,7 +362,6 @@ jobs:
     working_directory: ~/ddev
     environment:
       DDEV_DEBUG: "true"
-      ARTIFACTS: /artifacts
     steps:
       - checkout
       - run: sudo mkdir /home/linuxbrew && sudo chown $(id -u) /home/linuxbrew
@@ -388,44 +384,46 @@ jobs:
       # We only build the xz version of the docker images on tag build.
       - run:
           # Do not build the docker tarballs at simple tag build time
-          command: ./.circleci/generate_artifacts.sh $ARTIFACTS false false
+          command: ./.circleci/generate_artifacts.sh ~/artifacts false false
           name: tar/zip up artifacts and make hashes
           no_output_timeout: "40m"
 
       - store_artifacts:
-          path: /artifacts
+          path: ~/artifacts
           name: Artifact storage
 
   # 'release_build' is used to push a full release; it's triggered by api call
   release_build:
     macos:
-      xcode: "11.0.0"
+      xcode: "11.3.0"
     working_directory: ~/ddev
     environment:
       DDEV_DEBUG: "true"
-      ARTIFACTS: /artifacts
     steps:
     - checkout
     - restore_cache:
         keys:
-        - homebrew-macos-v13
+        - homebrew-macos-v14
     - run:
         command: ./.circleci/macos_circle_vm_setup.sh
-        name: RELEASE BUILD Circle VM setup
+        name: RELEASE BUILD (macOS) Circle VM setup
     - save_cache:
-        key: homebrew-macos-v13
+        key: homebrew-macos-v14
         paths:
         - /usr/local/Homebrew
         - /usr/local/Cellar
         - ~/Library/Caches/Homebrew
 
     - run:
-        command: make -s clean linux darwin_signed windows_install chocolatey
-        name: Build the ddev executables
+        command: make -s clean linux windows_install chocolatey
+
+    - run:
+        command: make -s darwin_notarized
+        no_output_timeout: 30m
 
     # We only build the xz version of the docker images on tag build.
     - run:
-        command: ./.circleci/generate_artifacts.sh $ARTIFACTS ${BUILD_IMAGE_TARBALLS:true}
+        command: ./.circleci/generate_artifacts.sh ~/artifacts ${BUILD_IMAGE_TARBALLS:true}
         name: tar/zip up artifacts and make hashes
         no_output_timeout: "40m"
 
@@ -441,13 +439,13 @@ jobs:
             -u $CIRCLE_PROJECT_USERNAME \
             -b "$(cat ./.github/RELEASE_NOTES_TEMPLATE.md)" \
             -t $GITHUB_TOKEN \
-            "${version}" $ARTIFACTS
+            "${version}" ~/artifacts
           else
             echo "GITHUB_TOKEN not provided, not pushing release $CIRCLE_TAG"
           fi
         name: Upload artifacts to GitHub release page
     - store_artifacts:
-        path: /artifacts
+        path: ~/artifacts
         name: Artifact storage
     # When fixed, this will have to be done after push to github, so it can use
     # the real github release artifact.

.circleci/linux_circle_vm_setup.sh

@@ -28,7 +28,7 @@ nvm use 10
 npm install --global markdownlint-cli
 markdownlint --version
 # readthedocs has ancient version of mkdocs in it.
-pip3 install mkdocs==0.17.5
+pip3 install yq mkdocs==0.17.5
 
 # Get the Stubs and Plugins for makensis; the linux makensis build doesn't do this.
 wget https://sourceforge.net/projects/nsis/files/NSIS%203/3.04/nsis-3.04.zip/download && sudo unzip -o -d /usr/local/share download && sudo mv /usr/local/share/nsis-3.04 /usr/local/share/nsis

.circleci/macos_circle_vm_setup.sh

@@ -8,8 +8,7 @@ DOCKER_URL=https://download.docker.com/mac/stable/31259/Docker.dmg
 curl -O -sSL $DOCKER_URL
 open -W Docker.dmg && cp -r /Volumes/Docker/Docker.app /Applications
 
-# Basic tools
-brew update >/dev/null 2>/dev/null
+export HOMEBREW_NO_AUTO_UPDATE=1
 
 # Get docker in first so we can install it and work on other things
 brew cask install ngrok
@@ -20,10 +19,12 @@ nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended &
 brew tap drud/ddev
 brew unlink python@2 || true
 
-brew install mysql-client zip makensis jq expect coreutils golang ddev mkcert osslsigncode ghr
-brew link mysql-client zip makensis jq expect coreutils golang ddev mkcert osslsigncode ghr
+brew install mysql-client zip makensis jq expect coreutils golang ddev mkcert osslsigncode ghr gnu-getopt
+brew link mysql-client zip makensis jq expect coreutils golang ddev mkcert osslsigncode ghr gnu-getopt
 
 brew link --force mysql-client
+# These links are required for osslsigncode to work
+brew link libgsf glib pcre
 
 # Get the Plugins for NSIS
 curl -fsSL -o /tmp/EnVar-Plugin.zip https://github.com/GsNSIS/EnVar/releases/latest/download/EnVar-Plugin.zip && sudo unzip -o -d /usr/local/share/nsis /tmp/EnVar-Plugin.zip
@@ -34,6 +35,8 @@ mkdir -p /usr/local/etc/my.cnf.d
 
 mkcert -install
 
+pip3 install yq
+
 curl -fsSL -o /tmp/gotestsum.tgz https://github.com/gotestyourself/gotestsum/releases/download/v0.3.2/gotestsum_0.3.2_darwin_amd64.tar.gz && tar -C /usr/local/bin -zxf /tmp/gotestsum.tgz gotestsum
 
 # gotestsum

.circleci/trigger_release.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-# trigger_release.sh --release-tag=v1.x.1 --circleci-token=token --github-token=githubPersonalToken --windows-signing-password=windowspass
+# .circleci/trigger_release.sh x --release-tag="v1.12.0-20" --circleci-token=circletoken  --build-image-tarballs=false --windows-signing-password=winsignpasswd --macos-signing-password=macsigningpwd --macos-app-password="macapppwd"
 
-# .circleci/trigger_release.sh --release-tag=v1.7.1 --circleci-token=circleToken900908b3443ea58316baf928b --github-token=githubPersonalToken853ae6f72c40525cd21036f742904a   --windows-signing-password=windowscodepassword | jq -r 'del(.circle_yml)'  | jq -r 'del(.circle_yml)'
+# .circleci/trigger_release.sh --release-tag=v1.11.1 --circleci-token=circletoken --github-token=githubtoken --build-image-tarballs=true --windows-signing-password=winsignpwd —macos-signing-password=macsignpwd —macos_app_password=macapppwd | jq -r 'del(.circle_yml)'
 
 # api docs: https://circleci.com/docs/api
 # Trigger a new job: https://circleci.com/docs/api/v1-reference/#new-build
@@ -27,53 +27,55 @@ if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
     exit 1
 fi
 
+LONGOPTS=circleci-token:,github-token:,release-tag:,github-project:,windows-signing-password:,macos-signing-password:,build-image-tarballs:,chocolatey-api-key:,github-org:,macos-app-password:
 
-OPTIONS=c:g:r:p:s:b:h:o:m:
-LONGOPTS=circleci-token:,github-token:,release-tag:,github-project:,windows-signing-password:,macos-signing-password:,build-image-tarballs:,chocolatey-api-key:,github-org:
-
-! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
+! PARSED=$(getopt --longoptions=$LONGOPTS --name "$0" -- "$@")
 if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
     # e.g. return value is 1
     #  then getopt has complained about wrong arguments to stdout
     printf "\n\nFailed parsing options:\n"
-    getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@"
+    getopt --longoptions=$LONGOPTS --name "$0" -- "$@"
     exit 2
 fi
 
 eval set -- "$PARSED"
 
 while true; do
     case "$1" in
-    -c|--circleci-token)
+    --circleci-token)
         CIRCLE_TOKEN=$2
         shift 2
         ;;
-    -g|--github-token)
+    --github-token)
         GITHUB_TOKEN=$2
         shift 2
         ;;
-    -t|--release-tag)
+    --release-tag)
         RELEASE_TAG=$2
         shift 2
         ;;
-    -p|--github-project)
+    --github-project)
         GITHUB_PROJECT=$2
         shift 2
         ;;
-    -s|--windows-signing-password)
+    --windows-signing-password)
         DDEV_WINDOWS_SIGNING_PASSWORD=$2
         shift 2
         ;;
-      -h|--chocolatey-api-key)
+    --chocolatey-api-key)
         CHOCOLATEY_API_KEY=$2
         shift 2
         ;;
-    -m|--macos-signing-password)
+    --macos-signing-password)
         DDEV_MACOS_SIGNING_PASSWORD=$2
         shift 2
         ;;
+    --macos-app-password)
+        DDEV_MACOS_APP_PASSWORD=$2
+        shift 2
+        ;;
     # For debugging we can set BUILD_IMAGE_TARBALLS=false to avoid waiting for that.
-    -b|--build-image-tarballs)
+    --build-image-tarballs)
         BUILD_IMAGE_TARBALLS=$2
         shift 2
         ;;
@@ -90,7 +92,7 @@ done
 trigger_build_url=https://circleci.com/api/v1.1/project/github/$GITHUB_PROJECT?circle-token=${CIRCLE_TOKEN}
 
 set -x
-BUILD_PARAMS="\"CIRCLE_JOB\": \"release_build\", \"job_name\": \"release_build\", \"GITHUB_TOKEN\":\"${GITHUB_TOKEN:-}\", \"RELEASE_TAG\": \"${RELEASE_TAG}\",\"DDEV_WINDOWS_SIGNING_PASSWORD\":\"${DDEV_WINDOWS_SIGNING_PASSWORD:-}\",\"DDEV_MACOS_SIGNING_PASSWORD\":\"${DDEV_MACOS_SIGNING_PASSWORD:-}\",\"CHOCOLATEY_API_KEY\":\"${CHOCOLATEY_API_KEY:-}\",\"BUILD_IMAGE_TARBALLS\":\"${BUILD_IMAGE_TARBALLS:-true}\",\"GITHUB_ORG\":\"${GITHUB_ORG}\""
+BUILD_PARAMS="\"CIRCLE_JOB\": \"release_build\", \"job_name\": \"release_build\", \"GITHUB_TOKEN\":\"${GITHUB_TOKEN:-}\", \"RELEASE_TAG\": \"${RELEASE_TAG}\",\"DDEV_WINDOWS_SIGNING_PASSWORD\":\"${DDEV_WINDOWS_SIGNING_PASSWORD:-}\",\"DDEV_MACOS_SIGNING_PASSWORD\":\"${DDEV_MACOS_SIGNING_PASSWORD:-}\",\"DDEV_MACOS_APP_PASSWORD\":\"${DDEV_MACOS_APP_PASSWORD:-}\",\"CHOCOLATEY_API_KEY\":\"${CHOCOLATEY_API_KEY:-}\",\"BUILD_IMAGE_TARBALLS\":\"${BUILD_IMAGE_TARBALLS:-true}\",\"GITHUB_ORG\":\"${GITHUB_ORG}\""
 if [ "${RELEASE_TAG:-}" != "" ]; then
     DATA="\"tag\": \"$RELEASE_TAG\","
 fi

Makefile

@@ -77,7 +77,7 @@ include build-tools/makefile_components/base_build_go.mak
 #include build-tools/makefile_components/base_test_go.mak
 #include build-tools/makefile_components/base_test_python.mak
 
-.PHONY: test testcmd testpkg build setup staticrequired windows_install darwin_signed markdownlint mkdocs
+.PHONY: test testcmd testpkg build setup staticrequired windows_install darwin_signed darwin_notarized markdownlint mkdocs
 
 TESTOS = $(shell uname -s | tr '[:upper:]' '[:lower:]')
 
@@ -129,17 +129,16 @@ mkdocs:
 
 darwin_signed: darwin
 	@if [ -z "$(DDEV_MACOS_SIGNING_PASSWORD)" ] ; then echo "Skipping signing ddev for macOS, no DDEV_MACOS_SIGNING_PASSWORD provided"; else echo "Signing macOS ddev..."; \
-		security create-keychain -p "$(DDEV_MACOS_SIGNING_PASSWORD)" buildagent; \
-		security list-keychains -s buildagent; \
-		security unlock-keychain -p "$(DDEV_MACOS_SIGNING_PASSWORD)" buildagent; \
-		security default-keychain -s buildagent; \
-		security import certfiles/macos_ddev_cert.p12 -k buildagent -P "$(DDEV_MACOS_SIGNING_PASSWORD)" -T /usr/bin/codesign >/dev/null ; \
-		security set-key-partition-list -S apple-tool:,apple: -s -k "$(DDEV_MACOS_SIGNING_PASSWORD)" buildagent >/dev/null ; \
-		codesign --keychain buildagent -s "Apple Distribution: DRUD Technology, LLC (3BAN66AG5M)" $(GOTMP)/bin/darwin_amd64/ddev ; \
-		security delete-keychain buildagent ; \
-		codesign -v $(GOTMP)/bin/darwin_amd64/ddev ; \
+		set -o errexit pipefail; \
+		curl -s https://raw.githubusercontent.com/drud/signing_tools/master/macos_sign.sh | bash -s -  --signing-password="$(DDEV_MACOS_SIGNING_PASSWORD)" --cert-file=certfiles/ddev_developer_id_cert.p12 --cert-name="Developer ID Application: DRUD Technology, LLC (3BAN66AG5M)" --target-binary="$(GOTMP)/bin/darwin_amd64/ddev" ; \
 	fi
 
+darwin_notarized: darwin_signed
+	@if [ -z "$(DDEV_MACOS_APP_PASSWORD)" ]; then echo "Skipping notarizing ddev for macOS, no DDEV_MACOS_APP_PASSWORD provided"; else \
+		set -o errexit pipefail; \
+		echo "Notarizing macOS ddev..." ; \
+		curl -s https://raw.githubusercontent.com/drud/signing_tools/master/macos_notarize.sh | bash -s -  --app-specific-password=${DDEV_MACOS_APP_PASSWORD} --apple-id=accounts@drud.com --primary-bundle-id=com.ddev.ddev --target-binary="$(PWD)/$(GOTMP)/bin/darwin_amd64/ddev" ; \
+	fi
 
 $(GOTMP)/bin/windows_amd64/ddev.exe: windows