Add support for integration with CloudBuild

Dockerfile

@@ -0,0 +1,4 @@
+FROM golang:1.13
+
+ADD . /go/src/github.com/m-lab/gcp-config
+RUN go get -v github.com/m-lab/gcp-config/cmd/stctl

cmd/stctl/main.go

@@ -18,6 +18,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/m-lab/gcp-config/internal/stctl"
 	"github.com/m-lab/gcp-config/transfer"
@@ -30,12 +31,13 @@ import (
 )
 
 var (
-	project      string
-	sourceBucket string
-	destBucket   string
-	prefixes     flagx.StringArray
-	startTime    flagx.Time
-	afterDate    flagx.DateTime
+	project         string
+	sourceBucket    string
+	destBucket      string
+	allowedProjects string
+	prefixes        flagx.StringArray
+	startTime       flagx.Time
+	afterDate       flagx.DateTime
 )
 
 func init() {
@@ -45,6 +47,7 @@ func init() {
 	flag.Var(&prefixes, "include", "Only transfer files with given prefix. Default all prefixes. Can be specified multiple times.")
 	flag.Var(&startTime, "time", "Start daily transfer at this time (HH:MM:SS)")
 	flag.Var(&afterDate, "after", "Only list operations that ran after the given date. Default is all dates.")
+	flag.StringVar(&allowedProjects, "allowed-projects", "", "If specified, exit when the current -project is not found in allowed-projects. Default is allow all.")
 }
 
 var usageText = `
@@ -88,6 +91,11 @@ func main() {
 	flag.Parse()
 	rtx.Must(flagx.ArgsFromEnv(flag.CommandLine), "Failed to parse flags")
 
+	if allowedProjects != "" && !strings.Contains(allowedProjects, project) {
+		// Exit cleanly.
+		os.Exit(0)
+	}
+
 	ctx := context.Background()
 	service, err := storagetransfer.NewService(ctx)
 	rtx.Must(err, "Failed to create new storage transfer service")

daily-archive-transfers.yaml

@@ -0,0 +1,50 @@
+options:
+  env:
+    # The -project flag should be read automatically from the environmnt.
+    - 'PROJECT=$PROJECT_ID'
+
+steps:
+# Create the gcp-config image for later steps.
+- name: gcr.io/cloud-builders/docker
+  args: [
+    'build', '-t', 'gcp-config', '.'
+  ]
+
+# 02:00:00 Nodes upload to the pusher-* bucket. 2hrs is the maximum upload delay.
+# 02:10:00 Configure daily pusher to local archive transfer.
+- name: gcp-config
+  args: [
+    'stctl', '-gcs.source=pusher-$PROJECT_ID',
+             '-gcs.target=archive-$PROJECT_ID',
+             '-time=02:10:00',
+             '-include=ndt',
+             '-include=host',
+             '-include=neubot',
+             '-include=utilization',
+             'sync'
+  ]
+  env:
+  - ALLOWED_PROJECTS=mlab-sandbox,mlab-staging,mlab-oti
+
+# 03:10:00 Configure daily local archive to public archive transfer.
+- name: gcp-config
+  args: [
+    'stctl', '-gcs.source=archive-mlab-oti',
+             '-gcs.target=archive-measurement-lab',
+             '-time=03:10:00',
+             'sync'
+  ]
+  env:
+  - ALLOWED_PROJECTS=measurement-lab
+
+# 04:10:00 Gardener or other jobs that depend on the public archive being up to date may run.
+# 04:10:00 Configure daily public archive to backup transfer.
+
+# NOTE: mlab-backups intentionally restricts access. This configuration is documentation.
+#- name: gcp-config
+#  args: [
+#    'stctl', '-gcs.source=archive-measurement-lab',
+#             '-gcs.target=mlab-cold-storage-backup',
+#             '-time=04:10:00',
+#             'sync'
+#  ]