From 0aa7f65c5a037e3ae3f7b5b79ed285d593b2f214 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Tue, 2 Apr 2024 11:33:30 -0700 Subject: [PATCH] Example of chromium using deployment scripts (#786) --- .../vocabularies/TraceMachina/accept.txt | 1 + deployment-examples/chromium/00_infra.sh | 1 + deployment-examples/chromium/01_operations.sh | 22 ++++ .../chromium/02_application.sh | 11 ++ .../chromium/03_build_chrome_tests.sh | 69 +++++++++++ .../chromium/04_delete_application.sh | 7 ++ deployment-examples/chromium/README.md | 76 ++++++++++++ deployment-examples/chromium/cas.json | 114 ++++++++++++++++++ deployment-examples/chromium/cas.yaml | 61 ++++++++++ .../example-do-not-use-in-prod-key.pem | 52 ++++++++ .../example-do-not-use-in-prod-rootca.crt | 29 +++++ deployment-examples/chromium/gateway.yaml | 24 ++++ .../chromium/kustomization.yaml | 23 ++++ deployment-examples/chromium/routes.yaml | 28 +++++ deployment-examples/chromium/scheduler.json | 93 ++++++++++++++ deployment-examples/chromium/scheduler.yaml | 51 ++++++++ .../chromium/worker-chromium.json | 80 ++++++++++++ .../chromium/worker-chromium.yaml | 35 ++++++ flake.nix | 17 +++ tools/pre-commit-hooks.nix | 1 + 20 files changed, 795 insertions(+) create mode 120000 deployment-examples/chromium/00_infra.sh create mode 100755 deployment-examples/chromium/01_operations.sh create mode 100755 deployment-examples/chromium/02_application.sh create mode 100755 deployment-examples/chromium/03_build_chrome_tests.sh create mode 100755 deployment-examples/chromium/04_delete_application.sh create mode 100644 deployment-examples/chromium/README.md create mode 100644 deployment-examples/chromium/cas.json create mode 100644 deployment-examples/chromium/cas.yaml create mode 100644 deployment-examples/chromium/example-do-not-use-in-prod-key.pem create mode 100644 deployment-examples/chromium/example-do-not-use-in-prod-rootca.crt create mode 100644 deployment-examples/chromium/gateway.yaml create mode 100644 deployment-examples/chromium/kustomization.yaml create mode 100644 deployment-examples/chromium/routes.yaml create mode 100644 deployment-examples/chromium/scheduler.json create mode 100644 deployment-examples/chromium/scheduler.yaml create mode 100644 deployment-examples/chromium/worker-chromium.json create mode 100644 deployment-examples/chromium/worker-chromium.yaml diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index e47410153..02d0ddc8a 100755 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -25,3 +25,4 @@ mutex parsable rebase remoteable +Chromium diff --git a/deployment-examples/chromium/00_infra.sh b/deployment-examples/chromium/00_infra.sh new file mode 120000 index 000000000..6183bee37 --- /dev/null +++ b/deployment-examples/chromium/00_infra.sh @@ -0,0 +1 @@ +../kubernetes/00_infra.sh \ No newline at end of file diff --git a/deployment-examples/chromium/01_operations.sh b/deployment-examples/chromium/01_operations.sh new file mode 100755 index 000000000..6cd0b699f --- /dev/null +++ b/deployment-examples/chromium/01_operations.sh @@ -0,0 +1,22 @@ +# This script configures a cluster with a few standard deployments. + +# TODO(aaronmondal): Add Grafana, OpenTelemetry and the various other standard +# deployments one would expect in a cluster. + +set -xeuo pipefail + +SRC_ROOT=$(git rev-parse --show-toplevel) + +kubectl apply -f ${SRC_ROOT}/deployment-examples/chromium/gateway.yaml + +# The image for the scheduler and CAS. +nix run .#image.copyTo \ + docker://localhost:5001/nativelink:local \ + -- \ + --dest-tls-verify=false + +# Wrap it with nativelink to turn it into a worker. +nix run .#nativelink-worker-siso-chromium.copyTo \ + docker://localhost:5001/nativelink-worker-siso-chromium:local \ + -- \ + --dest-tls-verify=false diff --git a/deployment-examples/chromium/02_application.sh b/deployment-examples/chromium/02_application.sh new file mode 100755 index 000000000..8cb689a41 --- /dev/null +++ b/deployment-examples/chromium/02_application.sh @@ -0,0 +1,11 @@ +# Get the nix derivation hash from the toolchain container, change the +# `TOOLCHAIN_TAG` variable in the `worker.json.template` to that hash and apply +# the configuration. + +KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/chromium + +kubectl apply -k "$KUSTOMIZE_DIR" + +kubectl rollout status deploy/nativelink-cas +kubectl rollout status deploy/nativelink-scheduler +kubectl rollout status deploy/nativelink-worker-chromium diff --git a/deployment-examples/chromium/03_build_chrome_tests.sh b/deployment-examples/chromium/03_build_chrome_tests.sh new file mode 100755 index 000000000..92d53007e --- /dev/null +++ b/deployment-examples/chromium/03_build_chrome_tests.sh @@ -0,0 +1,69 @@ +set -euo pipefail + +function fetch_chromium() { + mkdir -p ${HOME}/chromium + cd ${HOME}/chromium + fetch --no-history chromium +} + +# Based on requirements Ubuntu is the most well supported system +# https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md +if ! grep -q 'ID=ubuntu' /etc/os-release; then + echo "This system is not running Ubuntu." + exit 0 +fi + +if [ -d "${HOME}/chromium/src" ]; then + echo "Using existing chromium checkout" + cd ${HOME}/chromium + set +e + gclient sync --no-history + exit_status=$? + set -e + if [ $exit_status -ne 0 ]; then + echo "Failed to sync, removing files in ${HOME}/chromium" + rm -rf ${HOME}/chromium/ + fetch_chromium + fi + + cd src +else + echo "This script will modify the local system by adding depot_tools to .bashrc," + echo "downloading chrome code base and installing dependencies based on instructions" + echo "https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md." + echo "Do you want to continue? (yes/no)" + read answer + answer=$(echo "$answer" | tr '[:upper:]' '[:lower:]') + if [[ "$answer" != "yes" ]]; then + echo "Exiting." + # Exit or handle "no" logic here + exit 0 + fi + + # Add deport_tools to path + if [[ "$PATH" != *"/depot_tools"* ]]; then + cd ${HOME} + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo 'export PATH="${HOME}/depot_tools:$PATH"' >> ${HOME}/.bashrc + source ${HOME}/.bashrc + fi + + # Checkout chromium into home directory without history + fetch_chromium + cd src + + # Install dependencies required for clients to have on chromium builds + ./build/install-build-deps.sh +fi + +echo "Generating ninja projects" +gn gen --args="use_remoteexec=true rbe_cfg_dir=\"../../buildtools/reclient_cfgs/linux\"" out/Default + +# Fetch cache and schedular IP address for passing to ninja +CACHE=$(kubectl get gtw cache -o=jsonpath='{.status.addresses[0].value}') +SCHEDULER=$(kubectl get gtw scheduler -o=jsonpath='{.status.addresses[0].value}') +SCHEDULER_ADDRESS=${SCHEDULER}:50052 +CACHE_ADDRESS=${CACHE}:50051 + +echo "Starting autoninja build" +RBE_service=${SCHEDULER_ADDRESS} RBE_cas_service=${CACHE_ADDRESS} RBE_instance=main RBE_reclient_timeout=60m RBE_exec_timeout=4m RBE_alsologtostderr=true RBE_service_no_security=true RBE_service_no_auth=true RBE_local_resource_fraction=0.00001 RBE_automatic_auth=false RBE_gcert_refresh_timeout=20 RBE_compression_threshold=-1 RBE_metrics_namespace=main RBE_platform= RBE_experimental_credentials_helper= RBE_experimental_credentials_helper_args= RBE_log_http_calls=true RBE_use_rpc_credentials=false RBE_exec_strategy=remote_local_fallback autoninja -v -j 50 -C out/Default cc_unittests diff --git a/deployment-examples/chromium/04_delete_application.sh b/deployment-examples/chromium/04_delete_application.sh new file mode 100755 index 000000000..2ff9dfb22 --- /dev/null +++ b/deployment-examples/chromium/04_delete_application.sh @@ -0,0 +1,7 @@ +# Get the nix derivation hash from the toolchain container, change the +# `TOOLCHAIN_TAG` variable in the `worker.json.template` to that hash and delete +# the configuration. + +KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/chromium + +kubectl delete -k "$KUSTOMIZE_DIR" diff --git a/deployment-examples/chromium/README.md b/deployment-examples/chromium/README.md new file mode 100644 index 000000000..a410f5af0 --- /dev/null +++ b/deployment-examples/chromium/README.md @@ -0,0 +1,76 @@ +# Chromium example + +This deployment sets up a 4-container deployment with separate CAS, scheduler +and worker. Don't use this example deployment in production. It's insecure. + +> [!WARN] +> The client build request is best done from a Ubuntu image, `./03_build_chrome_tests.sh` +> will check if the image is Ubuntu and fail otherwise. + +All commands should be run from nix to ensure all dependencies exist in the environment. + +```bash +nix develop +``` + +In this example we're using `kind` to set up the cluster `cilium` to provide a +`LoadBalancer` and `GatewayController`. + +First set up a local development cluster: + +```bash +./00_infra.sh +``` + +Next start a few standard deployments. This part also builds the remote +execution containers and makes them available to the cluster: + +```bash +./01_operations.sh +``` + +Finally, deploy NativeLink: + +```bash +./02_application.sh +``` + +> [!TIP] +> You can use `./04_delete_application.sh` to remove just the `nativelink` +> deployments but leave the rest of the cluster intact. + +This demo setup creates two gateways to expose the `cas` and `scheduler` +deployments via your local docker network: + +```bash +CACHE=$(kubectl get gtw cache -o=jsonpath='{.status.addresses[0].value}') +SCHEDULER=$(kubectl get gtw scheduler -o=jsonpath='{.status.addresses[0].value}') + +echo "Cache IP: $CACHE" +echo "Scheduler IP: $SCHEDULER" +``` + +Using `./03_build_chrome_tests.sh` example script will download needed dependencies +for building Chromium unit tests using NativeLink CAS and Scheduler. The initial part +of the script checks if some dependencies exist, if not installs them, then moves on +to downloading and building Chromium tests. The script simplifies the setup described +in [linux/build_instructions.md](https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md) + +```bash +./03_build_chrome_tests.sh +``` + +> [!TIP] +> You can monitor the logs of container groups with `kubectl logs`: +> ```bash +> kubectl logs -f -l app=nativelink-cas +> kubectl logs -f -l app=nativelink-scheduler +> kubectl logs -f -l app=nativelink-worker-chromium --all-containers=true +> watch $HOME/chromium/src/buildtools/reclient/reproxystatus +> ``` + +When you're done testing, delete the cluster: + +```bash +kind delete cluster +``` diff --git a/deployment-examples/chromium/cas.json b/deployment-examples/chromium/cas.json new file mode 100644 index 000000000..2d742f14c --- /dev/null +++ b/deployment-examples/chromium/cas.json @@ -0,0 +1,114 @@ +// This configuration places objects in various directories in +// `~/.cache/nativelink`. When this location is mounted as a PersistentVolume +// it persists the cache across restarts. +{ + "stores": { + "CAS_MAIN_STORE": { + "existence_cache": { + "backend": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + } + } + } + } + }, + "AC_MAIN_STORE": { + "completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } + } + } + }, + "servers": [{ + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_MAIN_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "capabilities": {}, + "bytestream": { + "cas_stores": { + "main": "CAS_MAIN_STORE", + }, + } + } + }, + { + // Only publish metrics on a private port. + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50071", + "tls": { + "cert_file": "/root/example-do-not-use-in-prod-rootca.crt", + "key_file": "/root/example-do-not-use-in-prod-key.pem" + } + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_MAIN_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "capabilities": {}, + "bytestream": { + "cas_stores": { + "main": "CAS_MAIN_STORE", + } + } + } + }] +} diff --git a/deployment-examples/chromium/cas.yaml b/deployment-examples/chromium/cas.yaml new file mode 100644 index 000000000..96fe18331 --- /dev/null +++ b/deployment-examples/chromium/cas.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nativelink-cas +spec: + replicas: 1 + selector: + matchLabels: + app: nativelink-cas + template: + metadata: + labels: + app: nativelink-cas + spec: + containers: + - name: nativelink-cas + image: "localhost:5001/nativelink:local" + env: + - name: RUST_LOG + value: info + ports: + - containerPort: 50051 + - containerPort: 50061 + - containerPort: 50071 + volumeMounts: + - name: cas-config + mountPath: /cas.json + subPath: cas.json + - name: tls-volume + mountPath: /root + readOnly: true + args: ["/cas.json"] + volumes: + - name: cas-config + configMap: + name: cas + - name: tls-volume + secret: + secretName: tls-secret +--- +apiVersion: v1 +kind: Service +metadata: + name: nativelink-cas +spec: + selector: + app: nativelink-cas + ports: + - name: http + protocol: TCP + port: 50051 + targetPort: 50051 + - name: metrics + protocol: TCP + port: 50061 + targetPort: 50061 + - name: https + protocol: TCP + port: 50071 + targetPort: 50071 diff --git a/deployment-examples/chromium/example-do-not-use-in-prod-key.pem b/deployment-examples/chromium/example-do-not-use-in-prod-key.pem new file mode 100644 index 000000000..9ad6b97a0 --- /dev/null +++ b/deployment-examples/chromium/example-do-not-use-in-prod-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCWkpK9CeY3B1Q+ ++f5qTcavKEVPgJ4PqFdr9OJtOmboAcw9B9FcgM8QQ8FYkjwFYNE03ec0xCAUdPXG +gPXYxdU9ogwFYlTldV4Zhm9G7qbKLKWUE9rhXJHoiQZeLVHKUIn4UPSkMBr2UdQw +BVazG3RtJzRKTDfNpP6UAAmHlf/6MYWdQZwpK/Oelr4pCKofcavuiqmOyZmKEhpw +Nt0/6pFvLqERbaveuX2I+Ugvg0oY8voMueGAgT7qeE+ph+7BXQkaTBzYr+2p5TGy +aNXczXvkbL53KUxtLZ3bVlfLymGv8XpG8IVLroIOXbeQFSJpFiALilvZdtx2MnLk +hqiz193RNCVji9gL2l6JQDsL4L40XPY7iwmov+ar5fkjq3ysgIzpjRjZZUbt9DjJ +Ercmntfx3nDlXZni6/BnqQ9LxJC4Q1ZoONP6uXRJEC65vdRBusCXmRzLhijgdLLD +tSxk/1MQYztOq17X3HoOSUdlOrSMAyTiOysNTLCB3hWz53d0UAx2Chk+9Xd4LnVl +fOQH81OdK/QhAF6gGkhRM9UATe13v1nKLao+CS6U0xAjMNcA0mfd2/pVKXO8Ieuv +NEhI8UQOFvGX+DlJVXQtLAGrwGoNHzl2zbyI/8k1Lwp0bMzhrg7UfA9mb62rNVSH +3S9fu7XmeVuOi/Sb6iNFAGgMqXZm+QIDAQABAoICACMXC31kljMS0S5QAltxvrDJ +OMXbWnqZp63wBuifqKW+7YL3lHHzTs4ijCeBBN3X/0lLwXsJjdRWoYReP20CRby0 +TArhLJnA7WoTa1FjgpkxKHFG40aL1TEU5dgCxcHZXR3yd+DfmRj4XvPF+ruI1Wwu +q7/43TRelDmXfKENcvcynUnVpXfu1zqJNM3JSCRF0GSqCfPNxatopqKqFouztars +efrezSP9pGTMe2cfOKBx15Ypp4TUGf54hVAj7p8/eESZoktZTJDAagGC7vwjIxFL +ymNoy3fSHqykBZfnlTZ0lyN7IAVQKN0COCoq0rccweEOoHMggSZGgYICY8RttTKV +ZazKK0H+23oTj6zilgdHw0LNSV3+ByZF0zD0Ss2SZG7LHZJxJ8MS18i3U7XUG999 +LsWPacrIo5C1tK33WHWYLIIlJZyOqmJbvrRLleE3UImo26rwYVYP2I+KE6F3SH8b +lRCGYEqITXXKSBRDcAo63Ooohy9ftqeVRPzQtLzFVPODjnV3Ho3fCP+LdfjRCvED +9rk5f/zJR2SabW9OLKrvXoU0Z7+Oa7RvHiqH2Tdx1kUDuZnJgjcMF2pIeOPcAls7 +pEOrV5lIesLWSJQwRy2qsM1tr7/DFmpohGs8WnUbHwdLjWkFJXtkmTWcY7Yl3tm4 +aUD/8L4VNvsrGBtLHVmVAoIBAQDIjMe1umF/DgAWpuw0RYgELAyQ+MNc8GNalPV0 +FyAV2Sjzf08rfauJQ8PxJFAxkeI49uB2iGrcz1iuMC4l2sKdEHRbpYSArbmHcHsX +Hvq/n6TX2RH2GXv4k29c7dmj19phJMbTZCGvTpj+watJelwmK0cP/OmEWdLd3nud +dl6zrwsxxu88G330fg7KPBJIoVtMoczeOJ58lvlPXM+W729/lUhnItwIOU/AVu3u +xYe6skhVtX3YG0VWjNgFLK0KgaXJ2x+vNlrxOIOczpGiByfqpaCPoCG6vbVul/Bp +mpsp9iuFzHP8Ym938MMCsw/rrkQiZqcvgNrhHXT4XgUFN0qvAoIBAQDANFInrVOQ +LDs0G+s0nE+GSdjhSh2Qmf0rARs78XFxsRE/ZkHAzCfGAi9vggXqXtu3zzkC50KM +gAYgE2aQ6KoYJl5OU49o6nr7dWGguKZRVnfvnyVjq7h5ZYVHJQYry48+4SSkHzyY +NaGDNlnbKfehLxeQ3fzHrzajFCLsVMRiBb1wDEw+pbH1D7eSf6j7cOcgaUn0kfI9 +usPsjEZxxHAAuDYH8z6RHspucx3s+UxBIcyrljF+TNbEJ70ntow2172rAu/4O5d7 +NmTWBvX3YDsth52dHN199t+61cgz1/1gG15QEzvj8LHkGVPAu/8yQLskeJwMSSVJ +gOdHSvl9t7LXAoIBAQDIRkZ+BbWtcybFeEaz4Kf3OqIF+FborisZ313LY/inuOi7 +WeyPCv3Noz2+x7vJ9NyMdIYt+Qd2oR33PagoJ3Fn9nGsv/s1oceAKOTTbx2LVXsC +DwVv1X1G/L0eeWx5wWqUzmw72GgHMJ968Z8DTs1NVLSvk6sDf8wjwzWBRx9Fo02z +lO1+AhpjmatdbGX54CFTwtTbKoGi/AXWqmGoYk3fhA386QQAtnMdKfKMGvjziWzR +IuCceodg6mAjsrzPnC8bCNd4WviwofZVFri6rRjArExOgeSNHzxbSCJO45WGbw6B +nG5LoWWdynKEdJs2ih4CmK7msnBilM8l5IZw8gmDAoIBAQCte0wf9ejzu4igawas +EN4SlnsENdJjnyoMc84yF6ZOeQTZVaHJtDu+FGDeY9yVA5OL36VUwomlqTReJUSx +TN+iNpduWSubBfGFIBjDaYbs6YANr3ae9PLn18MpSPi99NjRZ4OcA3m85MNoXFlU +YRfay1eY8VTko3hMT7OJ3qT22Ll49hCnhwUN7WbC+yj58pka/w26izS7lOSckKxQ +qX5yl9Jk8J+resA6WvtK1mWGcEx9H26C0jYTDM9FlhYOtkHCpj8Uriz3EEyJhfTi +mGAxozOXCOO+e2LLD4TJjo1q/qjs27916N2XxWh8EPOxVw3TSG2JBUh7hLa7+ach +Wvo3AoIBAAM9Z9MOkAMECNwbSqsKt6/8j7RS+474cLPqXwRsmh1KPv5P4WkCTmxB +TDjqP/JkytL/0V3Zexc+2MUNwBwt1kUXRhi1rkbpi+fmZ/Sp8jh4fwriSRNj5BKz +0soGd/1dGF57EhpZHMQvts86y9A94tdKrVUb1Wosvsua22xAgeumbYGkc9gFKAkG +BiplOruOMkJhEQMR3gdrDUuyNL5HqBCGXP7v7+Lfgh99aZIh/ophex69T2QxTylw +sYDOH4+gMkJsYHU4H2UKm4JdGDZKRlxctwf//N1B+Ot7AULM/OPtHPgt/JcWxpjn +tLDdTkO8oICsLdyBDetlzNQ+h2hXfuM= +-----END PRIVATE KEY----- diff --git a/deployment-examples/chromium/example-do-not-use-in-prod-rootca.crt b/deployment-examples/chromium/example-do-not-use-in-prod-rootca.crt new file mode 100644 index 000000000..1e50c5e20 --- /dev/null +++ b/deployment-examples/chromium/example-do-not-use-in-prod-rootca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFCTCCAvGgAwIBAgIUBpZ3cJ8onptpeD/v2TVympU03sQwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIzMTAyMjA2MzMxNloXDTI0MTAy +MTA2MzMxNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAlpKSvQnmNwdUPvn+ak3GryhFT4CeD6hXa/TibTpm6AHM +PQfRXIDPEEPBWJI8BWDRNN3nNMQgFHT1xoD12MXVPaIMBWJU5XVeGYZvRu6myiyl +lBPa4VyR6IkGXi1RylCJ+FD0pDAa9lHUMAVWsxt0bSc0Skw3zaT+lAAJh5X/+jGF +nUGcKSvznpa+KQiqH3Gr7oqpjsmZihIacDbdP+qRby6hEW2r3rl9iPlIL4NKGPL6 +DLnhgIE+6nhPqYfuwV0JGkwc2K/tqeUxsmjV3M175Gy+dylMbS2d21ZXy8phr/F6 +RvCFS66CDl23kBUiaRYgC4pb2XbcdjJy5Iaos9fd0TQlY4vYC9peiUA7C+C+NFz2 +O4sJqL/mq+X5I6t8rICM6Y0Y2WVG7fQ4yRK3Jp7X8d5w5V2Z4uvwZ6kPS8SQuENW +aDjT+rl0SRAuub3UQbrAl5kcy4Yo4HSyw7UsZP9TEGM7Tqte19x6DklHZTq0jAMk +4jsrDUywgd4Vs+d3dFAMdgoZPvV3eC51ZXzkB/NTnSv0IQBeoBpIUTPVAE3td79Z +yi2qPgkulNMQIzDXANJn3dv6VSlzvCHrrzRISPFEDhbxl/g5SVV0LSwBq8BqDR85 +ds28iP/JNS8KdGzM4a4O1HwPZm+tqzVUh90vX7u15nlbjov0m+ojRQBoDKl2ZvkC +AwEAAaNTMFEwHQYDVR0OBBYEFL2SbUDfjD72r9B1/V2v33992AoWMB8GA1UdIwQY +MBaAFL2SbUDfjD72r9B1/V2v33992AoWMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggIBAF6WiXP61VEfdclSpV24wZfg1bSDpHw9S6LG7fy+/0oizpFY +nQkc+M2x2i7LFE8BoJKop6l4VyJ0jgGf5wzRHfmHX4QVatHBmzfJKA6Vm471aRVx +NM4/VWFpVbflnqGjeYEZcStM+506lgKfJk2It7Kd6pCmVbNzYb8W5zlvuYeWtwaZ +mtSLrmgsXLBfIwZBJagOEdQiM0mrRMT5LkA+fjKs+mqjDQz/RaDLTsrWnCnQ354Z +SBycbrJoyfnfAf0YlOlX+s2sjOEeSUvcjyI6mcA0Osyzerw79bztyaF132rU0A6D +jQw22Yj8B4tdcuTjaThXjPrlvWNOfoWbIyiU8Fk7BEvy3YFeRJ9qW9nx2BOCn0/y +quvu4W5NreDeIGT6FSX836GJYkEdheUnVDiKsiZZmB8Xng2D52gkWv5LI2U7LB4D +sZqFkyYXBR2xwoWbV46j0WnbffloUvCTk9oKOJ7i4i3kCyoWccfVUp2kVCdRK3Ok +bq0cPkudq7srb8IsjJVaAISvmrPXQj1sByLVhTJZsNsMmACDV5+QK+4fQn5hpAhS +RIG803s/fdITWxTzE+j8IM6YOcEhmDQpzUxhNFKGdZATNv3FF/YZdg8dCs8hvdxq +y8nkIB354me8h7bchCTIpD7OlL4D+vwPmuVs7IHpiXZJXPzCP0gkMX8Ewt6W +-----END CERTIFICATE----- diff --git a/deployment-examples/chromium/gateway.yaml b/deployment-examples/chromium/gateway.yaml new file mode 100644 index 000000000..bc6bf5450 --- /dev/null +++ b/deployment-examples/chromium/gateway.yaml @@ -0,0 +1,24 @@ +# TODO(aaronmondal): There should just be a single gateway. But that's currently +# bugged: https://github.com/cilium/cilium/issues/29099 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: cache +spec: + gatewayClassName: cilium + listeners: + - name: cache + protocol: HTTP + port: 50051 +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: scheduler +spec: + gatewayClassName: cilium + listeners: + - name: scheduler + protocol: HTTP + port: 50052 diff --git a/deployment-examples/chromium/kustomization.yaml b/deployment-examples/chromium/kustomization.yaml new file mode 100644 index 000000000..506d857be --- /dev/null +++ b/deployment-examples/chromium/kustomization.yaml @@ -0,0 +1,23 @@ +--- +resources: + - cas.yaml + - scheduler.yaml + - worker-chromium.yaml + - routes.yaml + +configMapGenerator: + - name: cas + files: + - cas.json + - name: scheduler + files: + - scheduler.json + - name: worker-chromium + files: + - worker-chromium.json + +secretGenerator: + - name: tls-secret + files: + - example-do-not-use-in-prod-rootca.crt + - example-do-not-use-in-prod-key.pem diff --git a/deployment-examples/chromium/routes.yaml b/deployment-examples/chromium/routes.yaml new file mode 100644 index 000000000..e094f9d02 --- /dev/null +++ b/deployment-examples/chromium/routes.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute # TODO(aaronmondal): Use GRPCRoute after resolution of + # https://github.com/TraceMachina/nativelink/issues/481 +metadata: + name: cache-route +spec: + parentRefs: + - sectionName: cache + name: cache + rules: + - backendRefs: + - name: nativelink-cas + port: 50051 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute # TODO(aaronmondal): Pure GRPC is unstable here. Find out why + # and migrate to a GRPCRoute. +metadata: + name: scheduler-route +spec: + parentRefs: + - sectionName: scheduler + name: scheduler + rules: + - backendRefs: + - name: nativelink-scheduler + port: 50052 diff --git a/deployment-examples/chromium/scheduler.json b/deployment-examples/chromium/scheduler.json new file mode 100644 index 000000000..c95cfe8f3 --- /dev/null +++ b/deployment-examples/chromium/scheduler.json @@ -0,0 +1,93 @@ +{ + "stores": { + "GRPC_LOCAL_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "cas" + } + }, + "GRPC_LOCAL_AC_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "ac" + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + // TODO(adams): use the right scheduler because reclient doesn't use the cached results? + // TODO(adams): max_bytes_per_stream + "simple": { + "supported_platform_properties": { + "cpu_count": "priority", + "memory_kb": "priority", + "network_kbps": "priority", + "disk_read_iops": "priority", + "disk_read_bps": "priority", + "disk_write_iops": "priority", + "disk_write_bps": "priority", + "shm_size": "priority", + "gpu_count": "priority", + "gpu_model": "priority", + "cpu_vendor": "priority", + "cpu_arch": "priority", + "cpu_model": "priority", + "kernel_version": "priority", + "OSFamily": "priority", + "container-image": "priority", + } + } + } + }, + "servers": [{ + "listener": { + "http": { + "socket_address": "0.0.0.0:50052" + } + }, + "services": { + "ac": { + "main": { + "ac_store": "GRPC_LOCAL_AC_STORE" + } + }, + "execution": { + "main": { + "cas_store": "GRPC_LOCAL_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061", + } + }, + "services": { + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + } + } + }] +} diff --git a/deployment-examples/chromium/scheduler.yaml b/deployment-examples/chromium/scheduler.yaml new file mode 100644 index 000000000..02a6892ac --- /dev/null +++ b/deployment-examples/chromium/scheduler.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nativelink-scheduler +spec: + replicas: 1 + selector: + matchLabels: + app: nativelink-scheduler + template: + metadata: + labels: + app: nativelink-scheduler + spec: + containers: + - name: nativelink-scheduler + image: "localhost:5001/nativelink:local" + env: + - name: RUST_LOG + value: info + - name: CAS_ENDPOINT + value: nativelink-cas + ports: + - containerPort: 50052 + volumeMounts: + - name: scheduler-config + mountPath: /scheduler.json + subPath: scheduler.json + args: ["/scheduler.json"] + volumes: + - name: scheduler-config + configMap: + name: scheduler +--- +apiVersion: v1 +kind: Service +metadata: + name: nativelink-scheduler +spec: + selector: + app: nativelink-scheduler + ports: + - name: scheduler + protocol: TCP + port: 50052 + targetPort: 50052 + - name: worker-api + protocol: TCP + port: 50061 + targetPort: 50061 diff --git a/deployment-examples/chromium/worker-chromium.json b/deployment-examples/chromium/worker-chromium.json new file mode 100644 index 000000000..7358bc5a2 --- /dev/null +++ b/deployment-examples/chromium/worker-chromium.json @@ -0,0 +1,80 @@ +// TODO(adam): add max file descriptors setting. +{ + "stores": { + "GRPC_LOCAL_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "cas" + } + }, + "GRPC_LOCAL_AC_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "ac" + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "~/.cache/nativelink/data-worker-test/content_path-cas", + "temp_path": "~/.cache/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + }, + "slow": { + "ref_store": { + "name": "GRPC_LOCAL_STORE", + } + } + } + } + }, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://${SCHEDULER_ENDPOINT:-127.0.0.1}:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "GRPC_LOCAL_AC_STORE", + }, + "work_directory": "~/.cache/nativelink/work", + "platform_properties": { + "cpu_count": { + "query_cmd": "nproc", + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": ["Linux"] + }, + "container-image": { + "values": ["placeholder"] + }, + } + } + }], + "servers": [], + "global": { + "max_open_files": 524288 + } +} diff --git a/deployment-examples/chromium/worker-chromium.yaml b/deployment-examples/chromium/worker-chromium.yaml new file mode 100644 index 000000000..cbc626f14 --- /dev/null +++ b/deployment-examples/chromium/worker-chromium.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nativelink-worker-chromium +spec: + replicas: 3 + selector: + matchLabels: + app: nativelink-worker-chromium + template: + metadata: + labels: + app: nativelink-worker-chromium + spec: + containers: + - name: nativelink-worker-chromium + image: "localhost:5001/nativelink-worker-siso-chromium:local" + env: + - name: RUST_LOG + value: info + - name: CAS_ENDPOINT + value: nativelink-cas + - name: SCHEDULER_ENDPOINT + value: nativelink-scheduler + volumeMounts: + - name: worker-chromium-config + mountPath: /worker-chromium.json + subPath: worker-chromium.json + command: ["/bin/nativelink"] + args: ["/worker-chromium.json"] + volumes: + - name: worker-chromium-config + configMap: + name: worker-chromium diff --git a/flake.nix b/flake.nix index 5f6cf94c3..141972b66 100644 --- a/flake.nix +++ b/flake.nix @@ -135,10 +135,26 @@ generate-toolchains = import ./tools/generate-toolchains.nix {inherit pkgs;}; + # inherit (nix2container.packages.${system}.nix2container) buildImage; + # rbe-autogen = import ./local-remote-execution/rbe-autogen.nix {inherit pkgs nativelink buildImage;}; + # createWorker = import ./tools/create-worker.nix {inherit pkgs nativelink buildImage;}; + + inherit (nix2container.packages.${system}.nix2container) pullImage; inherit (nix2container.packages.${system}.nix2container) buildImage; rbe-autogen = import ./local-remote-execution/rbe-autogen.nix {inherit pkgs nativelink buildImage;}; createWorker = import ./tools/create-worker.nix {inherit pkgs nativelink buildImage self;}; + siso-chromium = buildImage { + name = "siso-chromium"; + fromImage = pullImage { + imageName = "gcr.io/chops-public-images-prod/rbe/siso-chromium/linux"; + imageDigest = "sha256:26de99218a1a8b527d4840490bcbf1690ee0b55c84316300b60776e6b3a03fe1"; + sha256 = "sha256-v2wctuZStb6eexcmJdkxKcGHjRk2LuZwyJvi/BerMyw="; + tlsVerify = true; + arch = "amd64"; + os = "linux"; + }; + }; in rec { _module.args.pkgs = import self.inputs.nixpkgs { inherit system; @@ -160,6 +176,7 @@ lre-java = import ./local-remote-execution/lre-java.nix {inherit pkgs buildImage;}; rbe-autogen-lre-java = rbe-autogen lre-java; nativelink-worker-lre-java = createWorker lre-java; + nativelink-worker-siso-chromium = createWorker siso-chromium; image = buildImage { name = "nativelink"; config = { diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index 5fd5e1de0..28e4ecf38 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -43,6 +43,7 @@ in { excludes ++ [ # Integration testfiles not intended for production. + "deployment-examples/chromium/example-do-not-use-in-prod-key.pem" "deployment-examples/docker-compose/example-do-not-use-in-prod-key.pem" "deployment-examples/kubernetes/example-do-not-use-in-prod-key.pem" ];