[BEAM-9889] Populate local instance of InfluxDB with data

.test-infra/metrics/docker-compose.yml

@@ -41,13 +41,15 @@ services:
     volumes:
       - beam-influxdb-storage:/var/lib/influxdb
     environment:
-      - INFLUXDB_DB=beam_test_metrics
-      - INFLUXDB_HTTP_AUTH_ENABLED=true
       - INFLUXDB_BIND_ADDRESS=:8088
-      - INFLUXDB_USER=worker
-      - INFLUXDB_USER_PASSWORD=<UserPasswordHere>
-      - INFLUXDB_ADMIN_USER=admin
-      - INFLUXDB_ADMIN_PASSWORD=<AdminPasswordHere>
+  influxdb-restore:
+    build: ./influxdb
+    depends_on:
+      - influxdb
+    container_name: restore-backup
+    entrypoint: "sh"
+    command: ["-c", "./wait-for-it.sh influxdb:8086 -t 60 && \
+              influxd restore -portable -host influxdb:8088 /backup"]
   grafana:
     image: beamgrafana
     build: ./grafana
@@ -70,8 +72,6 @@ services:
       - INFLUX_DB_HOST=influxdb
       - INFLUX_DB_PORT=8086
       - INFLUX_DB_DBNAME=beam_test_metrics
-      - INFLUX_DB_USER=worker
-      - INFLUX_DB_PASSWORD=<UserPasswordHere>
   syncgithub:
     image: syncgithub
     container_name: beamsyncgithub

.test-infra/metrics/influxdb/Dockerfile

@@ -0,0 +1,35 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+FROM python:3-slim
+
+RUN pip install --no-cache-dir gsutil
+
+WORKDIR /
+
+RUN gsutil cp gs://apache-beam-testing-metrics/influxdb-backup.tar.gz . && \
+tar xzf influxdb-backup.tar.gz
+
+
+FROM influxdb:1.8.0
+
+ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /wait-for-it.sh
+
+RUN chmod +x /wait-for-it.sh
+
+COPY --from=0 /backup /backup

.test-infra/metrics/kubernetes/beam-influxdb-backups-persistentvolumeclaim.yaml → .test-infra/metrics/influxdb/gsutil/.boto

@@ -16,15 +16,9 @@
 # limitations under the License.
 ################################################################################
 
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: influxdb-backups
-  labels:
-    app: beammetrics
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi
+[GSUtil]
+default_project_id = apache-beam-testing
+default_api_version = 2
+
+[GoogleCompute]
+service_account = default

.test-infra/metrics/influxdb/gsutil/Dockerfile

@@ -0,0 +1,25 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+FROM python:3-slim
+
+# google-compute-engine package allows to obtain credentials for service
+# account specified in .boto file.
+RUN pip install --no-cache-dir gsutil google-compute-engine
+
+ADD .boto /etc/boto.cfg

.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml

@@ -16,35 +16,51 @@
 # limitations under the License.
 ################################################################################
 
-# A Cron Job that makes a full InfluxDB backup every week.
+# A Cron Job that makes a full InfluxDB backup every day and sends it to
+# GCS bucket.
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
   labels:
     app: beammetrics
   name: influxdb-autobackup
 spec:
-  schedule: "@weekly"
+  schedule: "@daily"
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
       template:
         spec:
-          containers:
-            - image: influxdb:1.8.0
-              name: influxdb-autobackup
+          # Using "initContainers" ensures that "create_backups" completes
+          # before "copy-to-gcs-bucket" begins.
+          initContainers:
+            - name: create-backup
+              image: influxdb:1.8.0
               args:
                 - influxd
                 - backup
                 - -portable
                 - -host
                 - influxdb-rpc:8088
-                - /backups
+                - /backup
+              volumeMounts:
+                - mountPath: /backup
+                  name: shared-data
+            - name: copy-to-gsc-bucket
+              image: gcr.io/apache-beam-testing/gsutil
+              command: ['sh', '-c', 'tar czf influxdb-backup.tar.gz /backup
+                        && gsutil cp influxdb-backup.tar.gz
+                        gs://apache-beam-testing-metrics/']
               volumeMounts:
-                - mountPath: /backups
-                  name: influxdb-backups
+                - mountPath: /backup
+                  name: shared-data
+                  readOnly: true
+          containers:
+            - name: job-done
+              image: busybox
+              command: ['sh', '-c' ,
+                        'echo "create-backup and copy-to-gsc-bucket completed"']
           restartPolicy: Never
           volumes:
-            - name: influxdb-backups
-              persistentVolumeClaim:
-                claimName: influxdb-backups
+            - name: shared-data
+              emptyDir: {}