NFS is broken due to firewall policies, disable it

bazelbuild · Aug 6, 2019 · b53d73f · b53d73f
1 parent 75fd577
commit b53d73f
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 67 deletions.
diff --git a/buildkite/create_images.py b/buildkite/create_images.py
@@ -45,7 +45,6 @@
         "source_image_project": "ubuntu-os-cloud",
         "source_image_family": "ubuntu-1804-lts",
         "setup_script": "setup-docker.sh",
-        "boot_disk_size": "10GB",
         "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"],
         "licenses": [
             "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
@@ -57,7 +56,6 @@
         "source_image_project": "ubuntu-os-cloud",
         "source_image_family": "ubuntu-1804-lts",
         "setup_script": "setup-docker.sh",
-        "boot_disk_size": "10GB",
         "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"],
         "licenses": [
             "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
@@ -69,7 +67,6 @@
         "source_image_project": "ubuntu-os-cloud",
         "source_image_family": "ubuntu-1804-lts",
         "setup_script": "setup-docker.sh",
-        "boot_disk_size": "10GB",
         "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"],
         "licenses": [
             "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"

diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py
@@ -52,7 +52,10 @@ def gcloud(*args, **kwargs):
         else:
             cmd += ["--" + flag, str(value)]
     # Throws `subprocess.CalledProcessError` if the process exits with return code > 0.
-    debug("Running: " + " ".join(cmd))
+
+    if not "get-serial-port-output" in cmd:
+        debug("Running: " + " ".join(cmd))
+
     return subprocess.run(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True
     )

diff --git a/buildkite/setup-docker-centos.sh b/buildkite/setup-docker-centos.sh
@@ -25,23 +25,6 @@ set -euxo pipefail
 ### Install base packages.
 {
     yum -y upgrade
-    yum -y install nfs-utils
-}
-
-### Add our Cloud Filestore volume to the fstab.
-{
-  case $(hostname -f) in
-    *.bazel-public.*)
-      cat >> /etc/fstab <<'EOF'
-10.93.166.218:/buildkite /opt nfs defaults,ro 0 2
-EOF
-      ;;
-    *.bazel-untrusted.*)
-      cat >> /etc/fstab <<'EOF'
-10.76.94.74:/buildkite /opt nfs defaults,ro 0 2
-EOF
-;;
-  esac
 }
 
 ### Increase file descriptor limits

diff --git a/buildkite/setup-docker.sh b/buildkite/setup-docker.sh
@@ -27,9 +27,9 @@ export DEBIAN_FRONTEND="noninteractive"
 
 ### Install base packages.
 {
-  apt-get -qqy update
-  apt-get -qqy dist-upgrade
-  apt-get -qqy install python nfs-common
+  apt-get -y update
+  apt-get -y dist-upgrade
+  apt-get -y install python openjdk-8-jdk unzip
 }
 
 ### Disable automatic upgrades, as they can interfere with our startup scripts.
@@ -39,22 +39,6 @@ APT::Periodic::Enable "0";
 EOF
 }
 
-### Add our Cloud Filestore volume to the fstab.
-{
-  case $(hostname -f) in
-    *.bazel-public.*)
-      cat >> /etc/fstab <<'EOF'
-10.93.166.218:/buildkite /opt nfs defaults,ro 0 2
-EOF
-      ;;
-    *.bazel-untrusted.*)
-      cat >> /etc/fstab <<'EOF'
-10.76.94.74:/buildkite /opt nfs defaults,ro 0 2
-EOF
-;;
-  esac
-}
-
 ### Increase file descriptor limits
 {
   cat >> /etc/security/limits.conf <<'EOF'
@@ -68,8 +52,8 @@ EOF
   apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \
       --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198
   add-apt-repository -y "deb https://apt.buildkite.com/buildkite-agent stable main"
-  apt-get -qqy update
-  apt-get -qqy install buildkite-agent
+  apt-get -y update
+  apt-get -y install buildkite-agent
 
   # Disable the Buildkite agent service, as the startup script has to mount /var/lib/buildkite-agent
   # first.
@@ -78,13 +62,13 @@ EOF
 
 ### Install Docker.
 {
-  apt-get -qqy install apt-transport-https ca-certificates
+  apt-get -y install apt-transport-https ca-certificates
 
-  curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
   add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 
-  apt-get -qqy update
-  apt-get -qqy install docker-ce
+  apt-get -y update
+  apt-get -y install docker-ce
 
   # Allow everyone access to the Docker socket. Usually this would be insane from a security point
   # of view, but these are untrusted throw-away machines anyway, so the risk is acceptable.
@@ -101,7 +85,7 @@ EOF
 
 ### Setup KVM.
 {
-  apt-get -qqy install qemu-kvm
+  apt-get -y install qemu-kvm
 
   # Allow everyone access to the KVM device. As above, this would usually not be a good idea, but
   # these machines are untrusted anyway...
@@ -110,25 +94,79 @@ EOF
 
 ### Get rid of Ubuntu's snapd stuff and install the Google Cloud SDK the traditional way.
 {
-  apt-get -qqy remove --purge snapd
+  apt-get -y remove --purge snapd
   echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \
       tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-  apt-get -qqy install apt-transport-https ca-certificates
-  curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+  apt-get -y install apt-transport-https ca-certificates
+  curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
       apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-  apt-get -qqy update
-  apt-get -qqy install google-cloud-sdk
+  apt-get -y update
+  apt-get -y install google-cloud-sdk
 }
 
 # Preseed our Git mirrors.
 {
   mkdir -p /var/lib/gitmirrors
-  curl https://storage.googleapis.com/bazel-git-mirror/bazelbuild-mirror.tar | tar x -C /var/lib/gitmirrors --strip=1
+  curl -fsSL https://storage.googleapis.com/bazel-git-mirror/bazelbuild-mirror.tar | \
+      tar x -C /var/lib/gitmirrors --strip=1
   # gsutil -qm rsync -rd gs://bazel-git-mirror/mirrors/ /var/lib/gitmirrors/
   chown -R buildkite-agent:buildkite-agent /var/lib/gitmirrors
   chmod -R 0755 /var/lib/gitmirrors
 }
 
+# Install Swift toolchains.
+{
+  curl -fsSL https://swift.org/builds/swift-4.2.1-release/ubuntu1404/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu14.04.tar.gz | \
+      tar xz -C /opt
+  curl -fsSL https://swift.org/builds/swift-4.2.1-release/ubuntu1604/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu16.04.tar.gz | \
+      tar xz -C /opt
+  curl -fsSL https://swift.org/builds/swift-4.2.1-release/ubuntu1804/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-ubuntu18.04.tar.gz | \
+      tar xz -C /opt
+}
+
+# Install Go.
+{
+  mkdir /opt/go1.12.6.linux-amd64
+  curl -fsSL https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz | \
+      tar xz -C /opt/go1.12.6.linux-amd64 --strip=1
+}
+
+# Install Android NDK.
+{
+  cd /opt
+  curl -fsSL -o android-ndk.zip https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip
+  unzip android-ndk.zip > /dev/null
+  rm android-ndk.zip
+}
+
+# Install Android SDK.
+{
+  mkdir -p /opt/android-sdk-linux
+  cd /opt/android-sdk-linux
+  curl -fsSL -o android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip
+  unzip android-sdk.zip > /dev/null
+  rm android-sdk.zip
+  yes | tools/bin/sdkmanager --licenses > /dev/null || true
+  tools/bin/sdkmanager --update
+  tools/bin/sdkmanager \
+      "build-tools;27.0.3" \
+      "build-tools;28.0.2" \
+      "emulator" \
+      "extras;android;m2repository" \
+      "platform-tools" \
+      "platforms;android-24" \
+      "platforms;android-28" \
+      "system-images;android-19;default;x86" \
+      "system-images;android-21;default;x86" \
+      "system-images;android-22;default;x86" \
+      "system-images;android-23;default;x86"
+}
+
+# Fix permissions in /opt.
+{
+  chown -R root:root /opt
+}
+
 ### Clean up and trim the filesystem (potentially reduces the final image size).
 {
   rm -rf /var/lib/apt/lists/*

diff --git a/buildkite/startup-docker.sh b/buildkite/startup-docker.sh
@@ -48,20 +48,6 @@ mkdir -p "${AGENT_HOME}/.cache/bazel/_bazel_buildkite-agent"
 chown -R buildkite-agent:buildkite-agent "${AGENT_HOME}"
 chmod 0755 "${AGENT_HOME}"
 
-# Mount a shared NFS repository cache.
-REPOCACHE="/var/lib/buildkite-agent/.cache/bazel/_bazel_buildkite-agent/cache/repos"
-mkdir -p "${REPOCACHE}"
-case $(hostname -f) in
-  *.bazel-public.*)
-    mount -t nfs 10.112.185.98:/repocache "${REPOCACHE}"
-    ;;
-  *.bazel-untrusted.*)
-    mount -t nfs 10.163.211.226:/repocache "${REPOCACHE}"
-    ;;
-esac
-chown buildkite-agent:buildkite-agent "${REPOCACHE}"
-chmod 0755 "${REPOCACHE}"
-
 # Mount tmpfs to Docker's working directory.
 DOCKER_HOME="/var/lib/docker"
 mount --bind /tmpfs/docker "${DOCKER_HOME}"