[actions] Add build-and-test github action

.bazelrc

@@ -123,6 +123,7 @@ build:qemu-bpf --//bazel/test_runners:test_runner=qemu_with_kernel
 build:qemu-bpf --run_under="bazel/test_runners/qemu_with_kernel/test_runner.sh"
 build:qemu-bpf --build_tag_filters=requires_bpf,-no_qemu
 test:qemu-bpf --test_tag_filters=requires_bpf,-disabled,-no_qemu
+test:qemu-bpf --test_timeout=180,600,1800,3600
 
 # Build for GCC.
 # These are copts because they apply to both c++ and c files.
@@ -174,6 +175,7 @@ build:tsan --define tcmalloc=disabled
 # https://github.com/google/sanitizers/issues/953
 build:tsan --test_env=TSAN_OPTIONS=report_atomic_races=0
 build:tsan --features=tsan
+test:tsan --test_timeout=180,600,1800,3600
 
 # Note that we are lumping tests that require root into the BPF tests below
 # to minimize number of configs.

.github/actions/bazelrc/action.yaml

@@ -5,6 +5,9 @@ inputs:
   download_toplevel:
     description: 'whether to download cached toplevel files during bazel builds'
     default: 'false'
+  dev:
+    description: 'Whether to use DEV or CI settings for the bazelrc. defaults to dev'
+    default: 'true'
 runs:
   using: "composite"
   steps:
@@ -21,3 +24,14 @@ runs:
     run: |
       echo "build --remote_download_outputs=toplevel" >> github.bazelrc
     shell: bash
+  - name: Add DEV settings
+    if: inputs.dev == 'true'
+    run: |
+      echo "build --remote_upload_local_results=false" >> github.bazelrc
+      echo "build --build_metadata=ROLE=DEV" >> github.bazelrc
+    shell: bash
+  - name: Add CI settings
+    if: inputs.dev != 'true'
+    run: |
+      echo "build --build_metadata=ROLE=CI" >> github.bazelrc
+    shell: bash

.github/workflows/build_and_test.yaml

@@ -0,0 +1,135 @@
+---
+name: build-and-test
+on:
+  pull_request:
+  push:
+    branches:
+    - 'main'
+  schedule:
+  # Run at 23:09 PST (07:09 UTC) every day. Github suggests not running actions on the hour.
+  - cron: '9 7 * * *'
+permissions:
+  contents: read
+jobs:
+  get-dev-image:
+    uses: ./.github/workflows/get_image.yaml
+    with:
+      image-base-name: "dev_image_with_extras"
+  clang-tidy:
+    needs: get-dev-image
+    runs-on: [self-hosted, nokvm]
+    container:
+      image: ${{ needs.get-dev-image.outputs.image-with-tag }}
+      volumes:
+      - /etc/bazelrc:/etc/bazelrc
+      options: --cpus 16
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - name: get bazel config
+      uses: ./.github/actions/bazelrc
+    - name: Save Diff Info
+      run: ./ci/save_diff_info.sh
+    - name: Run Clang Tidy
+      shell: bash
+      # yamllint disable rule:indentation
+      run: |
+        diff_file="diff_origin_main_cc"
+        if [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ github.event_name }}" == "schedule" ]]; then
+          diff_file="diff_head_cc"
+        fi
+        ./ci/run_clang_tidy.sh -f "${diff_file}"
+      # yamllint enable rule:indentation
+  generate-matrix:
+    needs: get-dev-image
+    runs-on: [self-hosted, nokvm]
+    container:
+      image: ${{ needs.get-dev-image.outputs.image-with-tag }}
+      volumes:
+      - /etc/bazelrc:/etc/bazelrc
+      options: --cpus 16
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - name: get bazel config
+      uses: ./.github/actions/bazelrc
+    - name: Set matrix
+      id: set-matrix
+      shell: bash
+      run: |
+        echo '${{ toJSON(github) }}' > github.json
+        matrix="$(./ci/github/matrix.sh github.json)"
+        echo "Build & Test matrix: ${matrix}"
+        echo "matrix=${matrix}" >> $GITHUB_OUTPUT
+    - name: Upload Target Files
+      uses: actions/upload-artifact@v3
+      with:
+        name: target_files
+        path: |
+          bazel_buildables_*
+          bazel_tests_*
+  build-and-test:
+    needs: [get-dev-image, generate-matrix]
+    runs-on:
+    - self-hosted
+    - ${{ matrix.runner }}
+    container:
+      image: ${{ needs.get-dev-image.outputs.image-with-tag }}
+      volumes:
+      - /etc/bazelrc:/etc/bazelrc
+      - /var/run/docker.sock:/var/run/docker.sock
+      options: --privileged
+    if: needs.generate-matrix.outputs.matrix
+    strategy:
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
+      fail-fast: false
+    name: ${{ matrix.name }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - uses: actions/download-artifact@v3
+    - name: get dev bazel config
+      if: github.event_name != 'push'
+      uses: ./.github/actions/bazelrc
+      with:
+        dev: 'true'
+    - name: get ci bazel config
+      if: github.event_name == 'push'
+      uses: ./.github/actions/bazelrc
+      with:
+        dev: 'false'
+    - name: Build ${{ matrix.name }}
+      shell: bash
+      # yamllint disable rule:indentation
+      run: |
+        ./scripts/bazel_ignore_codes.sh build \
+          ${{ matrix.args }} \
+          --target_pattern_file=target_files/${{ matrix.buildables }} \
+          2> >(tee bazel_stderr)
+      # yamllint enable rule:indentation
+    - name: Test ${{ matrix.name }}
+      if: ${{ matrix.tests }}
+      shell: bash
+      # yamllint disable rule:indentation
+      run: |
+        # Github actions container runner creates a docker network without IPv6 support. We enable it manually.
+        sysctl -w net.ipv6.conf.lo.disable_ipv6=0
+        ./scripts/bazel_ignore_codes.sh test ${{ matrix.args }} --target_pattern_file=target_files/${{ matrix.tests }} \
+          2> >(tee bazel_stderr)
+      # yamllint enable rule:indentation
+    - name: Buildbuddy URL
+      if: always()
+      run: |
+        [[ -f bazel_stderr ]] || (echo "Error before bazel ran" && exit 1)
+        bb_url="$(cat bazel_stderr | grep -Po "(?<=Streaming build results to: ).*" | tail -n 1)"
+        echo "${bb_url}"
+      shell: bash

bazel/test_runners/qemu_with_kernel/run_qemu.sh

@@ -29,7 +29,7 @@ check_env_set QEMU_KERNEL_IMAGE
 check_env_set QEMU_DISK_BASE_RO
 
 QEMU_MEMORY=${QEMU_MEMORY:-4096M}
-QEMU_CPU_COUNT=${QEMU_CPU_COUNT:-1}
+QEMU_CPU_COUNT=${QEMU_CPU_COUNT:-2}
 
 # This needs to match what is in the exit c file.
 QEMU_EXIT_BASE="0xf4"

ci/bazel_build_deps.sh

@@ -103,7 +103,7 @@ function compute_targets() {
   for file in $(git diff --name-only "${commit_range}" ); do
     for pat in "${poison_patterns[@]}"; do
       if [[ "$file" =~ ${pat} ]]; then
-        echo "File ${file} with ${pat} modified. Triggering full build"
+        echo "File ${file} with ${pat} modified. Triggering full build" >&2
         targets=("//...")
         return 0
       fi
@@ -130,7 +130,7 @@ function check_bpf_trigger() {
   for file in $(git diff --name-only "${commit_range}" ); do
     for pat in "${bpf_patterns[@]}"; do
       if [[ "$file" =~ ${pat} ]]; then
-        echo "File ${file} with ${pat} modified. Triggering bpf targets"
+        echo "File ${file} with ${pat} modified. Triggering bpf targets" >&2
         run_bpf_targets=true
         return 0
       fi

ci/github/bazelrc

@@ -8,7 +8,6 @@ common --keep_going
 build --build_metadata=HOST=github-actions
 build --build_metadata=USER=github-actions
 build --build_metadata=REPO_URL=https://github.com/pixie-io/pixie
-build --build_metadata=ROLE=DEV
 build --build_metadata=VISIBILITY=PUBLIC
 build --verbose_failures
 
@@ -39,3 +38,10 @@ test --local_cpu_resources=14
 
 build --jobs 16
 test --jobs 16
+
+# For qemu, we allocate 46 cpus instead of 16.
+build:qemu-bpf --local_cpu_resources=46
+test:qemu-bpf --local_cpu_resources=46
+
+build:qemu-bpf --jobs=48
+test:qemu-bpf --jobs=48

ci/github/matrix.sh

@@ -0,0 +1,166 @@
+#!/bin/bash -e
+
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This script outputs a github actions matrix with each of the configs we want to build and test
+# It takes a file with the github context as its first argument: https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+github_context="$1"
+
+event_name="$(jq -r '.event_name' < "${github_context}")"
+pr_body="$(jq -r '.event.pull_request.body' < "${github_context}")"
+commit_message="$(git log "$(git merge-base origin/main HEAD)"..HEAD --format="%B")"
+combined_message="${pr_body} \n${commit_message}"
+
+nightly_regression_test_iterations="${NIGHTLY_REGRESSION_TEST_ITERATIONS:-5}"
+
+all_kernel_versions=(
+  "4.14.254"
+  "4.19.254"
+  "5.10.173"
+  "5.15.101"
+  "5.4.235"
+  "6.1.18"
+)
+default_kernel_versions=(
+  "4.14.254"
+  "6.1.18"
+)
+kernel_versions=( "${default_kernel_versions[@]}" )
+
+check_tag() {
+  tag="$1"
+  echo "${combined_message}" | grep -P "${tag}([^\-a-zA-Z]|$)" > /dev/null
+}
+
+build_deps_flags=()
+extra_bazel_args=()
+# Build and test everything and BPF for main runs.
+if [[ "${event_name}" == "push" ]]; then
+  echo "Main run. Building all targets and bpf" >&2
+  build_deps_flags+=("-a" "-b")
+elif [[ "${event_name}" == "schedule" ]]; then
+  echo "Nightly run. Building all targets and all kernel versions of bpf" >&2
+  build_deps_flags+=("-a" "-b")
+  extra_bazel_args+=("--runs_per_test=${nightly_regression_test_iterations}")
+  kernel_versions=( "${all_kernel_versions[@]}" )
+elif [[ "${event_name}" == "pull_request" ]]; then
+  # Ignore bazel dependency tracking and run all targets if #ci:ignore-deps is in the pr body/commit message.
+  if check_tag '#ci:ignore-deps'; then
+    echo "Found #ci:ignore-deps tag. Building all targets" >&2
+    build_deps_flags+=("-a")
+  fi
+  # Build/Test bpf targets if #ci:bpf-build{-all-kernels} is in the pr body/commit message.
+  if check_tag '#ci:bpf-build-all-kernels'; then
+    echo "Found #ci:bpf-build tag. Building bpf targets" >&2
+    build_deps_flags+=("-b")
+    kernel_versions=( "${all_kernel_versions[@]}" )
+  elif check_tag '#ci:bpf-build'; then
+    echo "Found #ci:bpf-build-all-kernels. Building bpf targets on all kernel versions" >&2
+    build_deps_flags+=("-b")
+  fi
+fi
+
+# Create the target pattern files, passthrough flags to bazel_build_deps.
+./ci/bazel_build_deps.sh "${build_deps_flags[@]}"
+# TODO(james): generate the target pattern files directly from ci/github/matrix.yaml instead of using bazel_build_deps,
+# so that we don't have to edit bazel_build_deps.sh for new configs.
+
+remove_targetless_configs() {
+  f="$1"
+  # Remove any configs that don't have buildables.
+  while read -r build_target_file;
+  do
+    if [ -s "${build_target_file}" ]; then
+      continue
+    fi
+    yq -i e 'del(.configs[] | select(.buildables=="'"${build_target_file}"'"))' "${f}"
+  done < <(yq e '.configs[].buildables' "${f}" | sort | uniq)
+
+  # Remove the `tests` key from any configs where the test target file is empty.
+  while read -r test_target_file;
+  do
+    if [ -s "${test_target_file}" ]; then
+      continue
+    fi
+    yq -i e 'del(.configs[].tests | select(.=="'"${test_target_file}"'"))' "${f}"
+  done < <(yq e '.configs[].tests' "${f}" | sort | uniq)
+}
+
+remove_disabled_configs() {
+  yq -i e 'del(.configs[] | select(.disabled))' "$1"
+  yq -i e 'del(.configs[].disabled)' "$1"
+}
+
+merge_yamls() {
+  # shellcheck disable=SC2016
+  yq eval-all '. as $item ireduce ({}; . *+ $item)' "$@"
+}
+
+extend_with_kernel_versions() {
+  f="$1"
+  kernel_matrix_configs="$(mktemp)"
+  no_kernel="$(mktemp)"
+  yq e 'del(.configs[] | select(.use_kernel_matrix != true))' "$f" > "${kernel_matrix_configs}"
+  yq e 'del(.configs[] | select(.use_kernel_matrix))' "$f" > "${no_kernel}"
+
+  version_files=()
+  for version in "${kernel_versions[@]}"
+  do
+    version_configs="$(mktemp)"
+    yq e '.configs[].args += " --//bazel/test_runners/qemu_with_kernel:kernel_version='"${version}"'"' "$kernel_matrix_configs" > "${version_configs}"
+    yq -i e '.configs[].name += " ('"${version}"')"' "${version_configs}"
+    version_files+=("${version_configs}")
+  done
+
+  merge_yamls "${version_files[@]}" "${no_kernel}" > "$f"
+  yq -i e 'del(.configs[].use_kernel_matrix)' "$f"
+
+  rm "${kernel_matrix_configs}"
+  rm "${no_kernel}"
+  for f in "${version_files[@]}"
+  do
+    rm "$f"
+  done
+}
+
+add_extra_bazel_args() {
+  f="$1"
+  if [[ "${#extra_bazel_args[@]}" -gt 0 ]]; then
+    yq -i e '.configs[].args += " '"${extra_bazel_args[*]}"'"' "$f"
+  fi
+}
+
+process_matrix() {
+  in="$1"
+  out="$2"
+
+  cp "${in}" "${out}"
+  remove_disabled_configs "${out}"
+  remove_targetless_configs  "${out}"
+  add_extra_bazel_args "${out}"
+  extend_with_kernel_versions "${out}"
+}
+
+matrix="$(mktemp)"
+process_matrix "ci/github/matrix.yaml" "${matrix}"
+
+
+if [[ "$(yq e '.configs|length' "${matrix}")" != 0 ]]; then
+  yq e '.include = .configs | del(.configs)' -I=0 -o=json "${matrix}"
+fi
+
+rm "${matrix}"