add project dependencies (#5464)

* add project dependencies

* fix

* fix

Dockerfile.pr-builder

@@ -108,4 +108,6 @@ RUN conda env create -f ./python/hail/dev-environment.yml && \
 # gcloud iam key files will be stored here
 VOLUME /secrets
 
+RUN pip3 install --no-cache-dir pyyaml
+
 WORKDIR /hail

env-setup.sh

@@ -67,7 +67,7 @@ fi
 kubectl version -c || gcloud components install kubectl
 kubectl version || gcloud container clusters get-credentials $CLUSTER_NAME --region us-central1-a
 
-for project in $(cat projects.txt)
+for project in $(cat projects.yaml | grep '^- project: ' | sed 's/^- project: //')
 do
     if [[ -e $project/environment.yml ]]
     then

hail-ci-build-image

@@ -1 +1 @@
-gcr.io/hail-vdc/hail-pr-builder:bec585e2a68611174c329dac052ec27ee9fca17fbe9e37618f7e114f58c277a9
+gcr.io/hail-vdc/hail-pr-builder:4add3ab6bbb110e33fc7ef3f080015ce688e1a4db530032e8ac89e0d21ccd3f5

hail-ci-build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-PROJECTS=$(cat projects.txt)
+PROJECTS=$(cat projects.yaml | grep '^- project: ' | sed 's/^- project: //')
 
 for project in $PROJECTS; do
     if [[ -e $project/hail-ci-build.sh ]]; then

hail-ci-deploy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-PROJECTS=$(cat projects.txt)
+PROJECTS=$(cat projects.yaml | grep '^- project: ' | sed 's/^- project: //')
 
 SHA=$(git rev-parse --short=12 HEAD)
 

project-changed.py

@@ -1,3 +1,4 @@
+import yaml
 import sys
 import subprocess
 
@@ -8,19 +9,13 @@
     ''', sys.argv[0])
     exit(1)
 
-projects = []
-with open('projects.txt', 'r') as f:
-    for  line in f:
-        line = line.strip()
-        if not line:
-            continue
-
-        projects.append(line)
+with open('projects.yaml', 'r') as f:
+    projects = {x['project']: x for x in yaml.safe_load(f)}
 
 orig_hash = sys.argv[1]
 target_project = sys.argv[2]
 
-if target_project not in projects:
+if target_project not in projects.keys():
     sys.stderr.write('unknown project: {}\n'.format(target_project))
     exit(1)
 
@@ -30,19 +25,32 @@
     sys.stderr.write('command exited with return code {}: {}'.format(proc.returncode, ' '.join(cmd)))
     exit(1)
 
+
 def get_project(line):
-    for project in projects:
+    for project in projects.keys():
         if line.startswith(project + '/'):
             return project
     return None
 
+
+target_and_dependencies = projects[target_project].get('dependencies', [])
+target_and_dependencies.append(target_project)
+target_and_dependencies = set(target_and_dependencies)
+
+IRRELEVANT_FILES = set([
+    'project-changed.py', 'projects.yaml', 'env-setup.sh', 'README.md',
+    'SECRETS.md', 'LICENSE', 'AUTHORS'])
+
 for line in proc.stdout.decode('utf-8').split('\n'):
     line = line.strip()
     if not line:
         continue
 
+    if line in IRRELEVANT_FILES:
+        continue
+
     line_project = get_project(line)
-    if line_project == target_project or line_project is None:
+    if line_project in target_and_dependencies or line_project is None:
         print('yes')
         exit(0)
 

projects.yaml

@@ -0,0 +1,19 @@
+- project: auth-gateway
+- project: batch
+- project: ci
+  dependencies: ["batch"]
+- project: cloudtools
+- project: etc
+- project: notebook
+- project: notebook2
+- project: gateway
+- project: hail
+- project: apiserver
+- project: image-fetcher
+- project: letsencrypt
+- project: pipeline
+  dependencies: ["batch"]
+- project: scorecard
+- project: site
+- project: upload
+- project: vdc