From 6525313e039cadae5d900de23932b413434dee8e Mon Sep 17 00:00:00 2001 From: Hien To Date: Mon, 9 Dec 2024 14:56:41 +0700 Subject: [PATCH 1/3] feat: add codesign for macos --- .github/workflows/python-package.yml | 107 +++++++++++++-------------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b26ae74a1..5149fdc98 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -2,7 +2,7 @@ name: Build and Package Python Library on: push: - branches: [ feat/python-package-ci ] + branches: [ feat/python-codesign ] workflow_dispatch: inputs: model_dir: @@ -38,15 +38,15 @@ jobs: # - os: "linux" # name: "amd64" # runs-on: "ubuntu-20-04-cuda-12-0" - # - os: "mac" - # name: "amd64" - # runs-on: "macos-selfhosted-12" - # - os: "mac" - # name: "arm64" - # runs-on: "macos-silicon" - - os: "windows" + - os: "mac" name: "amd64" - runs-on: "windows-cuda-12-0" + runs-on: "macos-selfhosted-12" + - os: "mac" + name: "arm64" + runs-on: "macos-13-arm" + # - os: "windows" + # name: "amd64" + # runs-on: "windows-cuda-12-0" steps: - name: Clone id: checkout @@ -66,13 +66,33 @@ jobs: with: python-version: "3.11" + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 + shell: bash + env: + CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + + - uses: apple-actions/import-codesign-certs@v2 + continue-on-error: true + if: runner.os == 'macOS' + with: + p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} + + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 + shell: bash + env: + NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} + - name: Install dependencies Windows if: runner.os == 'windows' shell: pwsh run: | - - python3 -m pip install --upgrade pip - python3 -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt + python3 -m pip install --upgrade pip + python3 -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt - name: Install dependencies Linux if: runner.os == 'linux' @@ -102,9 +122,8 @@ jobs: echo "Python path (where.exe): $pythonPath" $pythonFolder = Split-Path -Path "$pythonPath" -Parent echo "PYTHON_FOLDER=$pythonFolder" >> $env:GITHUB_ENV - copy "$pythonFolder\python*.*" "$pythonFolder\Scripts\" - + - name: prepare python package macos if : runner.os == 'macOs' run: | @@ -128,7 +147,22 @@ jobs: rm -rf $PYTHON_FOLDER/lib/python3.1 echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" - + + - name: Notary macOS Binary + if: runner.os == 'macOS' + run: | + codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python + codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 + curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b /usr/local/bin + cd engine/cortex + # Notarize the binary + quill notarize ${{env.PYTHON_FOLDER}}/bin/python + quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 + env: + QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} + QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} + QUILL_NOTARY_KEY: "/tmp/notary-key.p8" + - name: Upload Artifact #if : runner.os == 'windows' || runner.os == 'linux' uses: actions/upload-artifact@v4 @@ -143,43 +177,8 @@ jobs: run: | rm ${{env.PYTHON_FOLDER}}/Scripts/python*.* - codesign: - runs-on: macos-latest - needs: build-and-test - steps: - - name: checkout - uses: actions/checkout@v3 - - uses: apple-actions/import-codesign-certs@v2 + - name: Remove Keychain continue-on-error: true - with: - p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} - p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: ${{env.MODEL_NAME}}-mac-amd64 - path: ${{env.MODEL_NAME}}-mac-amd64 - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: ${{env.MODEL_NAME}}-mac-arm64 - path: ${{env.MODEL_NAME}}-mac-arm64 - - - run: | - find "${{env.MODEL_NAME}}-mac-amd64" \( -type f -perm +111 \) -exec codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; - find "${{env.MODEL_NAME}}-mac-arm64" \( -type f -perm +111 \) -exec codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: ${{env.MODEL_NAME}}-mac-amd64-signed - path: ${{env.MODEL_NAME}}-mac-amd64 - include-hidden-files: true - compression-level: 9 - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: ${{env.MODEL_NAME}}-mac-arm64-signed - path: ${{env.MODEL_NAME}}-mac-arm64 - include-hidden-files: true - compression-level: 9 \ No newline at end of file + if: always() && runner.os == 'macOS' + run: | + security delete-keychain signing_temp.keychain \ No newline at end of file From 7af135257f2ca0b0e07a37080200417b86efda92 Mon Sep 17 00:00:00 2001 From: Hien To Date: Mon, 9 Dec 2024 15:04:13 +0700 Subject: [PATCH 2/3] feat: add codesign for macos --- .github/workflows/python-package.yml | 53 +++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5149fdc98..53baa0290 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -43,7 +43,7 @@ jobs: runs-on: "macos-selfhosted-12" - os: "mac" name: "arm64" - runs-on: "macos-13-arm" + runs-on: "macos-silicon" # - os: "windows" # name: "amd64" # runs-on: "windows-cuda-12-0" @@ -148,13 +148,56 @@ jobs: echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" + - name: create plist file + if: runner.os == 'macOS' + run: | + cat << EOF > /tmp/entitlements.plist + + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + + + com.apple.security.app-sandbox + + com.apple.security.network.client + + com.apple.security.network.server + + com.apple.security.device.audio-input + + com.apple.security.device.microphone + + com.apple.security.device.camera + + com.apple.security.files.user-selected.read-write + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-dyld-environment-variables + + com.apple.security.cs.allow-executable-memory + + + + EOF + - name: Notary macOS Binary if: runner.os == 'macOS' run: | - codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python - codesign --force --entitlements="./engine/templates/macos/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 - curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b /usr/local/bin - cd engine/cortex + codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python + codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 + # Code sign all .so files and .dylib files + + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" \) -exec codesign --deep --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" {} \; + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.dylib" \) -exec codesign --deep --force -s "${{ secrets.DEVELOPER_ID }}" {} \; + + curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin # Notarize the binary quill notarize ${{env.PYTHON_FOLDER}}/bin/python quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 From e62015f42eedb65249852634b96a918db8c453af Mon Sep 17 00:00:00 2001 From: Hien To Date: Tue, 10 Dec 2024 09:03:56 +0700 Subject: [PATCH 3/3] fix: notary python zipped folder --- .github/workflows/python-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 53baa0290..f0d7553ce 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -2,7 +2,7 @@ name: Build and Package Python Library on: push: - branches: [ feat/python-codesign ] + branches: [ feat/python-package-ci ] workflow_dispatch: inputs: model_dir: @@ -30,7 +30,7 @@ env: jobs: build-and-test: runs-on: ${{ matrix.runs-on }} - timeout-minutes: 60 + timeout-minutes: 360 strategy: fail-fast: false matrix: @@ -193,14 +193,14 @@ jobs: codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 # Code sign all .so files and .dylib files - - find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" \) -exec codesign --deep --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" {} \; - find ${{env.PYTHON_FOLDER}} -type f \( -name "*.dylib" \) -exec codesign --deep --force -s "${{ secrets.DEVELOPER_ID }}" {} \; + + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin # Notarize the binary quill notarize ${{env.PYTHON_FOLDER}}/bin/python quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec quill notarize {} \; env: QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }}