[docker] Remove cached git repositories from docker image

.github/workflows/build-docker.yml

@@ -7,32 +7,25 @@ on:
     paths:
       - 'ci/docker/tizen/**'
       - 'DEPS'
+  workflow_dispatch:
+
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - name: Extract branch name
-      id: branch
-      run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/})"
-
     - uses: actions/checkout@v2
-
     - uses: docker/setup-buildx-action@v1
-
     - uses: docker/login-action@v1
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-
     - name: Build and push
       uses: docker/build-push-action@v2
       with:
         context: ci/docker/tizen
         file: ci/docker/tizen/Dockerfile
         push: true
-        build-args:
-          GIT_BRANCH=${{ steps.branch.outputs.name }}
         tags: |
           ghcr.io/${{ github.repository_owner }}/build-engine:latest

ci/docker/tizen/Dockerfile

@@ -1,53 +1,15 @@
-#################
-# Builder stage #
-#################
-
-FROM debian:buster-slim AS builder
-
-ARG GIT_BRANCH
+FROM ghcr.io/flutter-tizen/tizen-tools:latest
 
 RUN apt-get update && \
-    apt-get install -y git curl ca-certificates python && \
+    apt-get install -y git curl ca-certificates python python3 xz-utils pkg-config \
+                       libncurses5 libfreetype6-dev && \
     apt-get clean
 
 # Install depot tools.
 ENV DEPOT_TOOLS_PATH=/usr/share/depot_tools
 ENV PATH=$PATH:${DEPOT_TOOLS_PATH}
 RUN git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git ${DEPOT_TOOLS_PATH}
 
-# Sync engine and dependencies.
-WORKDIR /engine
-RUN if [ -n "$GIT_BRANCH" ]; then \
-        git clone --depth 1 -b $GIT_BRANCH https://github.com/flutter-tizen/engine src/flutter; \
-    else \
-        git clone --depth 1 https://github.com/flutter-tizen/engine src/flutter; \
-    fi
-
-RUN gclient config --name="src/flutter" --deps-file="DEPS" --unmanaged https://github.com/flutter-tizen/engine.git
-RUN gclient setdep --var=download_android_deps=False --deps-file=src/flutter/DEPS
-RUN gclient sync -f -D --no-history --shallow
-
-# Copy selected .git directories.
-RUN mkdir -p cache
-RUN for x in $(find src/third_party/ -name .git -a \( -wholename "*/dart/*" -o -wholename "*/skia/*" \)); do \
-        mkdir -p cache/$x; \
-        cp -fr $x/* cache/$x; \
-    done
-
-
-#############################
-# build-engine docker image #
-#############################
-
-FROM ghcr.io/flutter-tizen/tizen-tools:latest
-
-RUN apt-get update && \
-    apt-get install -y git curl ca-certificates python xz-utils pkg-config \
-                       libncurses5 libfreetype6-dev && \
-    apt-get clean
-
-COPY --from=builder /usr/share/depot_tools /usr/share/depot_tools
-COPY --from=builder /engine/cache /engine/cache
-ADD prepare-sync.sh /engine/tools/
-
-ENV PATH=$PATH:/usr/share/depot_tools:/engine/tools
+# Add engine building tools.
+ADD tools/* /engine/tools/
+ENV PATH=$PATH:/engine/tools

ci/docker/tizen/tools/build-engine.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright 2021 Samsung Electronics Co., Ltd. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+BUILD_MODE=$1
+BUILD_ARCH=$2
+BUILD_TRIPLE=$3
+
+if [[ -z "$BUILD_MODE" || -z "$BUILD_ARCH" || -z "$BUILD_TRIPLE" ]]; then
+    echo "Usage: $(basename "$0") <mode> <arch> <triple>"
+    exit 1
+fi
+
+if [[ -z "$TIZEN_TOOLS_PATH" ]]; then
+    TIZEN_TOOLS_PATH=/tizen_tools
+fi
+if [ ! -d "$TIZEN_TOOLS_PATH" ]; then
+    echo "No such directory: $TIZEN_TOOLS_PATH"
+    exit 1
+fi
+
+# FIXME: Remove unsupported options in BUILD.gn.
+sed -i 's/"-Wno-non-c-typedef-for-linkage",//g' src/build/config/compiler/BUILD.gn
+sed -i 's/"-Wno-psabi",//g' src/build/config/compiler/BUILD.gn
+
+# Run gn.
+src/flutter/tools/gn \
+    --target-os linux \
+    --linux-cpu $BUILD_ARCH \
+    --no-goma \
+    --target-toolchain "$TIZEN_TOOLS_PATH"/toolchains \
+    --target-sysroot "$TIZEN_TOOLS_PATH"/sysroot/$BUILD_ARCH \
+    --target-triple $BUILD_TRIPLE \
+    --runtime-mode $BUILD_MODE \
+    --enable-fontconfig \
+    --embedder-for-target \
+    --disable-desktop-embeddings \
+    --build-tizen-shell
+
+# Run ninja.
+ninja -C src/out/linux_${BUILD_MODE}_${BUILD_ARCH}

ci/docker/tizen/tools/cache-checksum.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Copyright 2021 Samsung Electronics Co., Ltd. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# CACHE SCHECKSUM #
+#
+# Ninja build uses the modification time (mtime) of files for the incremental
+# build. After `gclient sync`, however, the incremental build does not work
+# because all synchronized files are newly updated and the mtime of files will
+# be newer than the cached out files.
+# So it is necessary to record the checksum of all source files used for the
+# cache output, and restore the mtime for the changed files using this checksum.
+#
+# Usages: cache-checksum <restore|save> <cache-dir>
+#
+# save: records the checksums of all files in TARGET_DIRS.
+# restore: finds the changed files using the recorded checksums, and make the
+#          files to have a new mtime than the cached files.
+#
+
+set -e
+
+TARGET_DIRS="src/flutter src/third_party src/buildtools"
+
+COMMAND=$1
+CACHE_PATH=$2
+
+if [[ -z "$COMMAND" || -z "$CACHE_PATH" ]]; then
+    echo "Usage: $(basename "$0") <restore|save> <cache-dir>"
+    exit 1
+fi
+
+CHECKSUM_FILE=$CACHE_PATH/.checksum.log
+
+if [[ "$COMMAND" == "restore" ]]; then
+    if [ ! -f "$CHECKSUM_FILE" ]; then
+        echo "No $CHECKSUM_FILE file. skipped."
+        exit 0
+    fi
+
+    # Set mtime of files in $TARGET_DIRS to OLD time.
+    for d in $(find $TARGET_DIRS -type d -not -path "*/.git*"); do
+        touch -c -m -d @1600000000 $d/* &
+    done
+
+    # Set mtime of changed files to NEW time.
+    for f in $(sha1sum -c --quiet $CHECKSUM_FILE 2>/dev/null | grep FAILED | cut -d: -f1); do
+        if [ -f $f ]; then
+            echo "changed file: $f"
+            touch $f
+        fi
+    done
+
+elif [[ "$COMMAND" == "save" ]]; then
+    find $TARGET_DIRS -type f -not -path "*/.git*" -print0 | xargs -r0 sha1sum >$CHECKSUM_FILE
+else
+    echo "unknown command: $COMMAND"
+    exit 1
+fi

ci/docker/tizen/tools/gclient-prepare-sync.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Copyright 2021 Samsung Electronics Co., Ltd. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# GCLIENT-PREPARE-SYNC #
+#
+# The `gclient sync` command takes a lot of time because it downloads overly
+# many objects to perform builds. To reduce the sync time, this script provides
+# following two methods:
+#
+# 1. Removing unnecesary dependencies.
+#    The DEPS describes unnecessary repositories to build the flutter-tizen,
+#    such as android, ios, fuchsia related things. By removing thease repositories
+#    not to download, the sync time can be shortened.
+#
+# 2. Shallow clone repositories.
+#    Even the `--no-history` option is used, the `gclient sync` fetches all objects
+#    of each repository. In CI build situations, these objects are wasted time and
+#    storage space. Before run `gclient sync`, these repositories can be shallow
+#    fetched by using `fetch --depth 1` to avoid these wastes.
+#
+
+set -e
+
+DEPS=src/flutter/DEPS
+
+while [ $# -ne 0 ]; do
+    name=$1
+    case "$name" in
+    --reduce-deps)
+        FLAG_REDUCE_DEPS=true
+        ;;
+    --shallow-sync)
+        FLAG_SHALLOW_SYNC=true
+        ;;
+    --deps)
+        shift
+        DEPS=$1
+        ;;
+    -h | --help)
+        script_name="$(basename "$0")"
+        echo "Usage: $script_name [Options]"
+        echo "       $script_name -h|-?|--help"
+        echo ""
+        echo "Options:"
+        echo "  --deps [DEPS]       Set DEPS file path. Default is \`$DEPS\`."
+        echo "  --reduce-deps       Remove unnecessary dependencies from DEPS."
+        echo "  --shallow-sync      Checkout git repositories with a single depth."
+        exit 0
+        ;;
+    *)
+        echo "Unknown argument \`$name\`"
+        exit 1
+        ;;
+    esac
+
+    shift
+done
+
+if [ ! -f $DEPS ]; then
+    echo "Could not find DEPS file. \`$DEPS\`"
+    exit 1
+fi
+
+cat >.gclient <<EOF
+solutions = [
+  { "name"        : 'src/flutter',
+    "url"         : 'https://github.com/flutter-tizen/engine.git',
+    "deps_file"   : 'DEPS',
+    "managed"     : False,
+  },
+]
+EOF
+
+if [[ "$FLAG_REDUCE_DEPS" == "true" ]]; then
+    gclient setdep --var=download_android_deps=False --deps-file=$DEPS
+    sed -i -e '/src\/ios_tools/,+2d' $DEPS
+    sed -i -e '/src\/third_party\/vulkan/,+2d' $DEPS
+    sed -i -e '/src\/third_party\/angle/,+2d' $DEPS
+    sed -i -e '/src\/third_party\/abseil-cpp/,+2d' $DEPS
+    sed -i -e '/src\/fuchsia\/sdk\/linux/,+9d' $DEPS
+fi
+
+if [[ "$FLAG_SHALLOW_SYNC" == "true" ]]; then
+    export PYTHONPATH=$PYTHONPATH:/usr/share/depot_tools
+    python3 "$(dirname -- "$0")/gclient-shallow-sync.py" "$DEPS"
+fi

ci/docker/tizen/tools/gclient-shallow-sync.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import subprocess
+import concurrent.futures
+import gclient_eval
+import gclient_utils
+
+
+def run_git(args, cwd):
+    if not args:
+        raise Exception('Please provide a git command to run')
+    return subprocess.call(['git'] + args, cwd=cwd) == 0
+
+
+def checkout_shallow_git(path, url, revision):
+    """Checks out a shallow reference, then sets the expected branch to be
+    checked out."""
+
+    os.makedirs(path, exist_ok=True)
+    run_git(['init', '--quiet'], path)
+    run_git(['config', 'advice.detachedHead', 'false'], path)
+    run_git(['remote', 'add', 'origin', url], path)
+    run_git(['fetch', '--depth', '1', 'origin', revision], path)
+    run_git(['checkout', 'FETCH_HEAD'], path)
+
+
+def checkout_deps(deps):
+    """Checks out the shallow refs specified in the DEPS file."""
+    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+        futures = []
+        for deps_name, deps_data in deps.items():
+            if deps_data.get('dep_type') == 'git':
+                url = deps_data['url'].split('@')[0]
+                revision = deps_data['url'].split('@')[1]
+                futures.append(executor.submit(
+                    checkout_shallow_git, deps_name, url, revision))
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
+
+
+def main(argv):
+    if (len(argv) < 1):
+        raise Exception('Please provide a DEPS file to update')
+    deps_file = argv[0]
+    if not os.path.exists(deps_file):
+        raise Exception('DEPS file does not exist')
+    deps_contents = gclient_utils.FileRead(deps_file)
+    local_scope = gclient_eval.Parse(deps_contents, deps_file)
+    checkout_deps(local_scope['deps'])
+
+
+if '__main__' == __name__:
+    sys.exit(main(sys.argv[1:]))