From 57f72b33f45de1f2adb7aabbd94474385f1d0eb3 Mon Sep 17 00:00:00 2001 From: Wonyoung Choi Date: Tue, 14 Dec 2021 19:16:06 +0900 Subject: [PATCH 1/5] [ci] Add gen_snapshot artifacts --- .github/workflows/build.yml | 107 ++++++++++++++++++++++++++++++++++++ tools/gn | 9 +++ 2 files changed, 116 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfecdb4becffc..44cd05bda81e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,6 +101,113 @@ jobs: src/out/linux_release_arm/cpp_client_wrapper !src/out/linux_release_arm/cpp_client_wrapper/engine_method_result.cc + - uses: actions/upload-artifact@v2 + if: (matrix.arch == 'arm' || matrix.arch == 'arm64') && (matrix.mode == 'release' || matrix.mode == 'profile') + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_linux-x64 + path: + src/out/${{ env.OUTPUT_NAME }}/clang_x64/gen_snapshot + + windows-build: + runs-on: windows-2019 + + strategy: + matrix: + arch: [arm, arm64] + mode: [release, profile] + + steps: + - name: Checkout engine + run: | + mkdir C:\workspace\engine\src\flutter + cd C:\workspace\engine\src\flutter + git init --quiet + git remote add origin https://github.com/${{ github.repository }} + git fetch --depth 1 origin ${{ github.sha }} + git checkout FETCH_HEAD + + - name: Setup environments + run: | + Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -Force + echo "DEPOT_TOOLS_WIN_TOOLCHAIN=0" >> $Env:GITHUB_ENV + echo "GYP_MSVS_OVERRIDE_PATH=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise" >> $Env:GITHUB_ENV + echo "WINDOWSSDKDIR=C:\Program Files (x86)\Windows Kits\10" >> $Env:GITHUB_ENV + + - name: Install depot_tools + run: | + Invoke-WebRequest -Uri https://storage.googleapis.com/chrome-infra/depot_tools.zip -OutFile depot_tools.zip + 7z x -y -o"C:\workspace\depot_tools" .\depot_tools.zip + Remove-Item depot_tools.zip + echo "C:\workspace\depot_tools" >> $Env:GITHUB_PATH + + - name: Print environment variables + run: | + gci env:* + Get-PSDrive + + - name: GClient Sync + working-directory: C:\workspace\engine + run: | + gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine + gclient sync -v --no-history --shallow + + - name: Build + working-directory: C:\workspace\engine\src + run: | + python .\flutter\tools\gn --target-os=linux --linux-cpu=${{ matrix.arch }} --no-goma --runtime-mode=${{ matrix.mode }} + ninja -C .\out\linux_${{ matrix.mode }}_${{ matrix.arch }} gen_snapshot + + - uses: actions/upload-artifact@v2 + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_windows-x64 + path: | + C:\workspace\engine\src\out\linux_${{ matrix.mode }}_${{ matrix.arch }}\gen_snapshot.exe + + macos-build: + runs-on: macos-11 + + strategy: + matrix: + arch: [arm, arm64] + mode: [release, profile] + + steps: + - uses: actions/checkout@v2 + with: + path: src/flutter + + - name: Install depot_tools + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo "$PWD/depot_tools" >> $GITHUB_PATH + + - name: Print environments + run: | + printenv + + - name: GClient Sync + run: | + gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine + gclient setdep --var=download_android_deps=False --deps-file=src/flutter/DEPS + gclient sync -v --no-history --shallow + + - name: Build + working-directory: ./src + run: | + # Change host_toolchain to mac/clang_x64 + sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" build/config/BUILDCONFIG.gn + # Pass dummy toolchain values to use custom toolchain settings + python3 ./flutter/tools/gn --linux --linux-cpu=${{ matrix.arch }} --runtime-mode=${{ matrix.mode }} \ + --no-goma --macos-build-tizen \ + --target-toolchain _ --target-sysroot _ --target-triple _ + ninja -C ./out/linux_${{ matrix.mode }}_${{ matrix.arch }} clang_x64/gen_snapshot + + - uses: actions/upload-artifact@v2 + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_darwin-x64 + path: | + src/out/linux_${{ matrix.mode }}_${{ matrix.arch }}/clang_x64/gen_snapshot + test: needs: build runs-on: ubuntu-latest diff --git a/tools/gn b/tools/gn index f10bd922218c9..e7482a721f4bd 100755 --- a/tools/gn +++ b/tools/gn @@ -390,6 +390,13 @@ def to_gn_args(args): if args.target_os in ['android'] and gn_args['target_cpu'] in ['x64' , 'arm64']: gn_args['dart_use_compressed_pointers'] = True + # Overwrite arguments for building tizen files on MacOS. + if args.macos_build_tizen: + gn_args['allow_deprecated_api_calls'] = False + gn_args['skia_use_metal'] = False + gn_args['shell_enable_metal'] = False + gn_args['use_default_linux_sysroot'] = False + return gn_args def parse_args(args): @@ -496,6 +503,8 @@ def parse_args(args): 'true and is enabled on CI.') parser.add_argument('--no-prebuilt-dart-sdk', dest='prebuilt_dart_sdk', action='store_false') + parser.add_argument('--macos-build-tizen', default=False, action='store_true') + # Sanitizers. parser.add_argument('--asan', default=False, action='store_true') parser.add_argument('--lsan', default=False, action='store_true') From f76146c06a5552a819c9ed0dd159fc0ec9d9e998 Mon Sep 17 00:00:00 2001 From: Wonyoung Choi Date: Thu, 23 Dec 2021 10:24:16 +0900 Subject: [PATCH 2/5] Remove printing environment values --- .github/workflows/build.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44cd05bda81e1..ae7a753360e84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,14 +137,8 @@ jobs: run: | Invoke-WebRequest -Uri https://storage.googleapis.com/chrome-infra/depot_tools.zip -OutFile depot_tools.zip 7z x -y -o"C:\workspace\depot_tools" .\depot_tools.zip - Remove-Item depot_tools.zip echo "C:\workspace\depot_tools" >> $Env:GITHUB_PATH - - name: Print environment variables - run: | - gci env:* - Get-PSDrive - - name: GClient Sync working-directory: C:\workspace\engine run: | @@ -181,10 +175,6 @@ jobs: git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git echo "$PWD/depot_tools" >> $GITHUB_PATH - - name: Print environments - run: | - printenv - - name: GClient Sync run: | gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine @@ -194,9 +184,9 @@ jobs: - name: Build working-directory: ./src run: | - # Change host_toolchain to mac/clang_x64 + # Change host_toolchain to mac/clang_x64. sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" build/config/BUILDCONFIG.gn - # Pass dummy toolchain values to use custom toolchain settings + # Pass dummy toolchain values to use custom toolchain settings. python3 ./flutter/tools/gn --linux --linux-cpu=${{ matrix.arch }} --runtime-mode=${{ matrix.mode }} \ --no-goma --macos-build-tizen \ --target-toolchain _ --target-sysroot _ --target-triple _ From 9531e90ce84b4bbae2b0c7dc6acfffd7ff0e6ffd Mon Sep 17 00:00:00 2001 From: Wonyoung Choi Date: Thu, 23 Dec 2021 13:36:58 +0900 Subject: [PATCH 3/5] Apply shallow gclient sync --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae7a753360e84..e6aaef8c98a82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,6 +121,7 @@ jobs: run: | mkdir C:\workspace\engine\src\flutter cd C:\workspace\engine\src\flutter + git config --global core.autocrlf true git init --quiet git remote add origin https://github.com/${{ github.repository }} git fetch --depth 1 origin ${{ github.sha }} @@ -142,13 +143,15 @@ jobs: - name: GClient Sync working-directory: C:\workspace\engine run: | - gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine + gclient config --name=src\flutter --unmanaged https://github.com/flutter-tizen/engine + $env:PYTHONPATH="C:\workspace\depot_tools" + python3 src\flutter\ci\tizen\gclient-shallow-sync.py src\flutter\DEPS gclient sync -v --no-history --shallow - name: Build working-directory: C:\workspace\engine\src run: | - python .\flutter\tools\gn --target-os=linux --linux-cpu=${{ matrix.arch }} --no-goma --runtime-mode=${{ matrix.mode }} + python3 .\flutter\tools\gn --target-os=linux --linux-cpu=${{ matrix.arch }} --no-goma --runtime-mode=${{ matrix.mode }} ninja -C .\out\linux_${{ matrix.mode }}_${{ matrix.arch }} gen_snapshot - uses: actions/upload-artifact@v2 @@ -179,6 +182,7 @@ jobs: run: | gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine gclient setdep --var=download_android_deps=False --deps-file=src/flutter/DEPS + PYTHONPATH="$PWD/depot_tools" python3 ./src/flutter/ci/tizen/gclient-shallow-sync.py src/flutter/DEPS gclient sync -v --no-history --shallow - name: Build From 38257531799a1fe961eb97d99f9f635b7e8984c8 Mon Sep 17 00:00:00 2001 From: Wonyoung Choi Date: Thu, 23 Dec 2021 17:55:48 +0900 Subject: [PATCH 4/5] Apply cache to macos-build job --- .github/workflows/build.yml | 23 +++++++++++++++++++---- ci/tizen/cache-checksum.sh | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6aaef8c98a82..e93cf6a9e2a6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,15 +168,29 @@ jobs: arch: [arm, arm64] mode: [release, profile] + env: + OUTPUT_NAME: linux_${{ matrix.mode }}_${{ matrix.arch }} + steps: - uses: actions/checkout@v2 with: path: src/flutter + - uses: actions/cache@v2 + with: + path: src/out/${{ env.OUTPUT_NAME }} + key: out-macos-build-${{ env.OUTPUT_NAME }}-${{ github.sha }} + restore-keys: | + out-macos-build-${{ env.OUTPUT_NAME }}- + - name: Install depot_tools run: | git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git echo "$PWD/depot_tools" >> $GITHUB_PATH + + - name: Install required packages + run: | + brew install md5sha1sum - name: GClient Sync run: | @@ -186,15 +200,16 @@ jobs: gclient sync -v --no-history --shallow - name: Build - working-directory: ./src run: | + src/flutter/ci/tizen/cache-checksum.sh restore src/out/$OUTPUT_NAME # Change host_toolchain to mac/clang_x64. - sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" build/config/BUILDCONFIG.gn + sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" src/build/config/BUILDCONFIG.gn # Pass dummy toolchain values to use custom toolchain settings. - python3 ./flutter/tools/gn --linux --linux-cpu=${{ matrix.arch }} --runtime-mode=${{ matrix.mode }} \ + python3 ./src/flutter/tools/gn --linux --linux-cpu=${{ matrix.arch }} --runtime-mode=${{ matrix.mode }} \ --no-goma --macos-build-tizen \ --target-toolchain _ --target-sysroot _ --target-triple _ - ninja -C ./out/linux_${{ matrix.mode }}_${{ matrix.arch }} clang_x64/gen_snapshot + ninja -C ./src/out/$OUTPUT_NAME clang_x64/gen_snapshot + src/flutter/ci/tizen/cache-checksum.sh save src/out/$OUTPUT_NAME - uses: actions/upload-artifact@v2 with: diff --git a/ci/tizen/cache-checksum.sh b/ci/tizen/cache-checksum.sh index b1690e967cdb6..d24887ffd54a6 100755 --- a/ci/tizen/cache-checksum.sh +++ b/ci/tizen/cache-checksum.sh @@ -41,7 +41,7 @@ if [[ "$COMMAND" == "restore" ]]; then # Set mtime of files in $TARGET_DIRS to OLD time. for d in $(find $TARGET_DIRS -type d); do - touch -c -m -d @1600000000 $d/* & + touch -t 202001010000 $d/* & done # Set mtime of changed files to NEW time. From e32dbbb001df99d8fc0b9d971b8f6bec9a050ae3 Mon Sep 17 00:00:00 2001 From: Wonyoung Choi Date: Fri, 24 Dec 2021 13:07:49 +0900 Subject: [PATCH 5/5] Apply suggestions by code review --- .github/workflows/build.yml | 46 ++++++++++++++++++------------------- tools/gn | 11 +++------ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e93cf6a9e2a6c..c2eb697068f34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,15 +81,13 @@ jobs: if: matrix.os == 'host' with: name: host-${{ matrix.arch }}-${{ matrix.mode }} - path: | - src/out/${{ env.OUTPUT_NAME }}/*_unittests + path: src/out/${{ env.OUTPUT_NAME }}/*_unittests - uses: actions/upload-artifact@v2 if: matrix.os == 'linux' with: name: tizen-${{ matrix.arch }}-${{ matrix.mode }} - path: | - src/out/${{ env.OUTPUT_NAME }}/libflutter_*.so + path: src/out/${{ env.OUTPUT_NAME }}/libflutter_*.so - uses: actions/upload-artifact@v2 if: matrix.arch == 'arm' && matrix.mode == 'release' @@ -105,8 +103,7 @@ jobs: if: (matrix.arch == 'arm' || matrix.arch == 'arm64') && (matrix.mode == 'release' || matrix.mode == 'profile') with: name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_linux-x64 - path: - src/out/${{ env.OUTPUT_NAME }}/clang_x64/gen_snapshot + path: src/out/${{ env.OUTPUT_NAME }}/clang_x64/gen_snapshot windows-build: runs-on: windows-2019 @@ -117,7 +114,7 @@ jobs: mode: [release, profile] steps: - - name: Checkout engine + - name: checkout engine run: | mkdir C:\workspace\engine\src\flutter cd C:\workspace\engine\src\flutter @@ -127,20 +124,20 @@ jobs: git fetch --depth 1 origin ${{ github.sha }} git checkout FETCH_HEAD - - name: Setup environments + - name: setup environments run: | Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -Force echo "DEPOT_TOOLS_WIN_TOOLCHAIN=0" >> $Env:GITHUB_ENV echo "GYP_MSVS_OVERRIDE_PATH=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise" >> $Env:GITHUB_ENV echo "WINDOWSSDKDIR=C:\Program Files (x86)\Windows Kits\10" >> $Env:GITHUB_ENV - - name: Install depot_tools + - name: install depot_tools run: | Invoke-WebRequest -Uri https://storage.googleapis.com/chrome-infra/depot_tools.zip -OutFile depot_tools.zip 7z x -y -o"C:\workspace\depot_tools" .\depot_tools.zip echo "C:\workspace\depot_tools" >> $Env:GITHUB_PATH - - name: GClient Sync + - name: gclient sync working-directory: C:\workspace\engine run: | gclient config --name=src\flutter --unmanaged https://github.com/flutter-tizen/engine @@ -148,17 +145,19 @@ jobs: python3 src\flutter\ci\tizen\gclient-shallow-sync.py src\flutter\DEPS gclient sync -v --no-history --shallow - - name: Build + - name: build working-directory: C:\workspace\engine\src run: | - python3 .\flutter\tools\gn --target-os=linux --linux-cpu=${{ matrix.arch }} --no-goma --runtime-mode=${{ matrix.mode }} + python3 .\flutter\tools\gn ` + --linux --linux-cpu=${{ matrix.arch }} ` + --runtime-mode=${{ matrix.mode }} ` + --no-goma ninja -C .\out\linux_${{ matrix.mode }}_${{ matrix.arch }} gen_snapshot - uses: actions/upload-artifact@v2 with: name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_windows-x64 - path: | - C:\workspace\engine\src\out\linux_${{ matrix.mode }}_${{ matrix.arch }}\gen_snapshot.exe + path: C:\workspace\engine\src\out\linux_${{ matrix.mode }}_${{ matrix.arch }}\gen_snapshot.exe macos-build: runs-on: macos-11 @@ -183,30 +182,32 @@ jobs: restore-keys: | out-macos-build-${{ env.OUTPUT_NAME }}- - - name: Install depot_tools + - name: install depot_tools run: | git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git echo "$PWD/depot_tools" >> $GITHUB_PATH - - name: Install required packages + - name: install required packages run: | brew install md5sha1sum - - name: GClient Sync + - name: gclient sync run: | gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine gclient setdep --var=download_android_deps=False --deps-file=src/flutter/DEPS PYTHONPATH="$PWD/depot_tools" python3 ./src/flutter/ci/tizen/gclient-shallow-sync.py src/flutter/DEPS gclient sync -v --no-history --shallow - - name: Build + - name: build run: | src/flutter/ci/tizen/cache-checksum.sh restore src/out/$OUTPUT_NAME # Change host_toolchain to mac/clang_x64. sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" src/build/config/BUILDCONFIG.gn - # Pass dummy toolchain values to use custom toolchain settings. - python3 ./src/flutter/tools/gn --linux --linux-cpu=${{ matrix.arch }} --runtime-mode=${{ matrix.mode }} \ - --no-goma --macos-build-tizen \ + # Pass dummy toolchain values to avoid using linux target toolchain settings. + python3 ./src/flutter/tools/gn \ + --linux --linux-cpu=${{ matrix.arch }} \ + --runtime-mode=${{ matrix.mode }} \ + --no-goma \ --target-toolchain _ --target-sysroot _ --target-triple _ ninja -C ./src/out/$OUTPUT_NAME clang_x64/gen_snapshot src/flutter/ci/tizen/cache-checksum.sh save src/out/$OUTPUT_NAME @@ -214,8 +215,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_darwin-x64 - path: | - src/out/linux_${{ matrix.mode }}_${{ matrix.arch }}/clang_x64/gen_snapshot + path: src/out/linux_${{ matrix.mode }}_${{ matrix.arch }}/clang_x64/gen_snapshot test: needs: build diff --git a/tools/gn b/tools/gn index e7482a721f4bd..cfb64e57915ab 100755 --- a/tools/gn +++ b/tools/gn @@ -301,7 +301,7 @@ def to_gn_args(args): # attributes in release modes till the toolchain is updated. gn_args['skia_enable_api_available_macro'] = args.runtime_mode != "release" - if sys.platform == 'darwin' and args.target_os not in ['android', 'fuchsia']: + if sys.platform == 'darwin' and args.target_os not in ['android', 'fuchsia', 'linux']: # OpenGL is deprecated on macOS > 10.11. # This is not necessarily needed but enabling this until we have a way to # build a macOS metal only shell and a gl only shell. @@ -390,11 +390,8 @@ def to_gn_args(args): if args.target_os in ['android'] and gn_args['target_cpu'] in ['x64' , 'arm64']: gn_args['dart_use_compressed_pointers'] = True - # Overwrite arguments for building tizen files on MacOS. - if args.macos_build_tizen: - gn_args['allow_deprecated_api_calls'] = False - gn_args['skia_use_metal'] = False - gn_args['shell_enable_metal'] = False + # Don't use the default linux sysroot when buliding for Linux on MacOS + if sys.platform == 'darwin' and args.target_os == 'linux': gn_args['use_default_linux_sysroot'] = False return gn_args @@ -503,8 +500,6 @@ def parse_args(args): 'true and is enabled on CI.') parser.add_argument('--no-prebuilt-dart-sdk', dest='prebuilt_dart_sdk', action='store_false') - parser.add_argument('--macos-build-tizen', default=False, action='store_true') - # Sanitizers. parser.add_argument('--asan', default=False, action='store_true') parser.add_argument('--lsan', default=False, action='store_true')