Merge pull request #1 from apache/master

Update forked repository.

.test-infra/jenkins/PrecommitJobBuilder.groovy

@@ -68,7 +68,9 @@ class PrecommitJobBuilder {
       '^gradle.bat$',
       '^settings.gradle$'
     ]
-    triggerPathPatterns.addAll defaultPathTriggers
+    if (triggerPathPatterns) {
+      triggerPathPatterns.addAll defaultPathTriggers
+    }
     job.with {
       description buildDescription('for each commit push.')
       concurrentBuild()

.test-infra/jenkins/job_PostCommit_CommunityMetrics.groovy

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+
+job('beam_Prober_CommunityMetrics') {
+  description('Health check probes for the Community Metrics infrastructure')
+ commonJobProperties.setTopLevelMainJobProperties(delegate)
+
+ commonJobProperties.enablePhraseTriggeringFromPullRequest(delegate,
+     'Community Metrics Prober',
+     'Run Community Metrics Prober')
+
+ commonJobProperties.setAutoJob(delegate)
+
+ commonJobProperties.enablePhraseTriggeringFromPullRequest(
+         delegate,
+         'Community Metrics Prober',
+         'Run Community Metrics Prober')
+
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(commonJobProperties.checkoutDir)
+      tasks(':communityMetricsProber')
+      commonJobProperties.setGradleSwitches(delegate)
+    }
+  }
+}

.test-infra/jenkins/job_PreCommit_CommunityMetrics.groovy

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+    scope: this,
+    nameBase: 'Community Metrics',
+    gradleTask: ':communityMetricsPreCommit',
+    triggerPathPatterns: ['^.test-infra/metrics/.*$']
+)
+builder.build()
+

.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+    scope: this,
+    nameBase: 'Java_Examples_Dataflow',
+    gradleTask: ':javaExamplesDataflowPreCommit',
+    triggerPathPatterns: [
+      '^model/.*$',
+      '^sdks/java/.*$',
+      '^runners/google-cloud-dataflow/.*$',
+      '^examples/java/.*$',
+      '^release/.*$',
+    ]
+)
+builder.build {
+  publishers {
+    archiveJunit('**/build/test-results/**/*.xml')
+  }
+}

.test-infra/jenkins/job_PreCommit_RAT.groovy

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+    scope: this,
+    nameBase: 'RAT',
+    gradleTask: ':rat'
+)
+builder.build()

.test-infra/metrics/README.md

@@ -17,17 +17,23 @@
     under the License.
 -->
 # BeamMonitoring
-This folder contains files required to spin-up metrics dashboard for Beam.
+This folder contains files required to deploy the Beam Community metrics stack
+on your local machine.
 
-## Utilized technologies
-* [Grafana](https://grafana.com) as dashboarding engine.
-* PostgreSQL as underlying DB.
+This includes
+* Python scripts for ingesting data from sources (Jenkins, JIRA,
+  GitHub)
+* Postgres analytics database
+* [Grafana](https://grafana.com) dashboarding UI
 
-Approach utilized is to fetch data from corresponding system: Jenkins/Jira/GithubArchives/etc, put it into PostreSQL and fetch it to show in Grafana.
+All components run within Docker containers. These are composed together via
+docker-compose for local hosting, and Kubernetes for the production instance on
+GCP.
 
 ## Local setup
 
-Install docker
+Docker Compose is used to host the full metrics stack on your local machine.
+
 * install docker
     * https://docs.docker.com/install/#supported-platforms
 * install docker-compose
@@ -76,8 +82,24 @@ docker-compose build
 docker-compose up
 ```
 
+After running these commands, you can access the services running on your local
+machine:
+
+* Grafana: http://localhost:3000
+* Postgres DB: localhost:5432
+
+If you're deploying for the first time on your machine, follow instructions on
+how to manually [configure Grafana](#configuring-grafana).
+
+Grafana and Postgres containers persist data to Docker volumes, which will be
+restored on subsequent runs. To start from a clean state, you must also wipe out
+these volumes. (List volumes via `docker volume ls`)
+
 ## Kubernetes setup
 
+The cloud-hosted topology is composed via Kubernetes instead of docker-compose.
+Follow the steps below to re-deploy the production setup.
+
 1. Configure gcloud & kubectl
   * https://cloud.google.com/kubernetes-engine/docs/quickstart
 2. Configure PosgreSQL
@@ -119,8 +141,33 @@ kubectl set image deployment/beamgrafana container=<new_image_name>
 kubectl replace -f beamgrafana-deploy.yaml
 ```
 
-
-## Useful Kubernetes commands and hints
+## Configuring Grafana
+
+When you deploy a new Grafana instance, there is some one-time setup:
+
+1. Log-in at http://localhost:3000 with username `admin` and the value specified
+  for `GF_SECURITY_ADMIN_PASSWORD` in
+  [`docker-compose.yml`](docker-compose.yml).
+1. Add Postgres as a data source:
+    1. Click the 'Add data source' button.
+    1. Fill out the following config:
+        * **Name**: BeamPSQL
+        * **Type**: PostgreSQL
+        * **Host** beampostgresql:5432
+        * **Database**: beam\_metrics
+        * **User**: admin
+        * **Password**: `POSTGRES_PASSWORD` in
+          [`docker-compose.yml`](docker-compose.yml).
+        * **SSL Mode**: Disable
+1. Restore dashboards from config
+    1. In the Grafana sidebar, hover over the plus (+) and select 'Import'
+    1. Select 'Upload .json File', and select the first exported JSON dashboard
+       file in [dashboards/](dashboards)
+    1. Repeat for each of the remaining exported dashboards.
+
+## Appendix
+
+### Useful Kubernetes commands and hints
 ```sh
 # Get pods
 kubectl get pods
@@ -135,7 +182,7 @@ kubectl log <PodName> <ContainerName>
 https://github.com/kubernetes/kubernetes/issues/35054
 ```
 
-## Useful docker commands and hints
+### Useful docker commands and hints
 * Connect from one container to another
     * `curl <containername>:<port>`
 * Remove all containers/images/volumes

.test-infra/metrics/build.gradle

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: "base"
+apply plugin: "groovy"
+
+repositories {
+  mavenCentral()
+}
+
+// https://github.com/avast/gradle-docker-compose-plugin
+apply plugin: "com.avast.gradle.docker-compose"
+
+dependencies {
+  testCompile library.groovy.groovy_all
+  testCompile library.java.junit
+}
+
+task testMetricsStack {
+  doLast {
+    // TODO(BEAM-5837): Add some actual validation of the metrics stack
+    println "Hello world!"
+  }
+}
+dockerCompose.isRequiredBy(testMetricsStack)
+
+task preCommit {
+  dependsOn testMetricsStack
+}
+
+task checkProber {
+  dependsOn test
+}