Ensured that beta runs in TeamCity use only beta paths (GoogleCloudPl…

…atform#10025)

* Ensured that beta runs in TeamCity use only beta paths

* Added tests for sweeper package path

mmv1/third_party/terraform/.teamcity/CONTRIBUTION_GUIDE.md

@@ -17,6 +17,14 @@ Note: these instructions need to be tested and improved. Please contact @SarahFr
 You will need to install:
 * Java 17
     * `brew install openjdk@17`
+* Maven
+    * `brew install --ignore-dependencies maven`
+
+Add the following to `~/.zshrc` and reload your terminal:
+
+```
+export JAVA_HOME=/usr/local/Cellar/openjdk@17/17.0.9/libexec/openjdk.jdk/Contents/Home
+```
 
 
 ## Getting started
@@ -27,7 +35,7 @@ You will need to install:
 * Run `make validate` to check the code for both:
     * Errors that prevent the code building
     * Logical errors in TeamCity-specific logic, e.g. the need for unique identifiers for builds.
-* Run `make tests` to run the automated tests defined in `.teamcity/tests`
+* Run `make test` to run the automated tests defined in `.teamcity/tests`
 
 ## Rough description of the code base
 

mmv1/third_party/terraform/.teamcity/components/inputs/packages.kt

@@ -7,7 +7,7 @@
 
 package generated
 
-var PackagesList = mapOf(
+var PackagesListGa = mapOf(
     "envvar" to mapOf(
         "name" to "envvar",
         "displayName" to "Environment Variables",
@@ -50,19 +50,61 @@ var PackagesList = mapOf(
     )
 )
 
-var SweepersList = mapOf(
+var SweepersListGa = mapOf(
     "sweeper" to mapOf(
         "name" to "sweeper",
         "displayName" to "Sweeper",
         "path" to "./google/sweeper"
     )
 )
 
-fun GetPackageNameList(): List<String> {
-    var packageNameList: ArrayList<String> = arrayListOf()
-    PackagesList.forEach{ p ->
-        var packageName = p.value.getValue("name").toString()
-        packageNameList.add(packageName)
-    }
-    return packageNameList
-}
+var PackagesListBeta = mapOf(
+    "envvar" to mapOf(
+        "name" to "envvar",
+        "displayName" to "Environment Variables",
+        "path" to "./google-beta/envvar"
+    ),
+    "fwmodels" to mapOf(
+        "name" to "fwmodels",
+        "displayName" to "Framework Models",
+        "path" to "./google-beta/fwmodels"
+    ),
+    "fwprovider" to mapOf(
+        "name" to "fwprovider",
+        "displayName" to "Framework Provider",
+        "path" to "./google-beta/fwprovider"
+    ),
+    "fwresource" to mapOf(
+        "name" to "fwresource",
+        "displayName" to "Framework Resource",
+        "path" to "./google-beta/fwresource"
+    ),
+    "fwtransport" to mapOf(
+        "name" to "fwtransport",
+        "displayName" to "Framework Transport",
+        "path" to "./google-beta/fwtransport"
+    ),
+    "provider" to mapOf(
+        "name" to "provider",
+        "displayName" to "SDK Provider",
+        "path" to "./google-beta/provider"
+    ),
+    "transport" to mapOf(
+        "name" to "transport",
+        "displayName" to "Transport",
+        "path" to "./google-beta/transport"
+    ),
+    "google" to mapOf(
+        "name" to "google",
+        "displayName" to "Google",
+        "path" to "./google-beta"
+    )
+)
+
+var SweepersListBeta = mapOf(
+    "sweeper" to mapOf(
+        "name" to "sweeper",
+        "displayName" to "Sweeper",
+        "path" to "./google-beta/sweeper"
+    )
+)

mmv1/third_party/terraform/.teamcity/components/projects/project_sweeper_project.kt

@@ -12,7 +12,7 @@ import SharedResourceNameBeta
 import SharedResourceNameGa
 import SharedResourceNameVcr
 import builds.*
-import generated.SweepersList
+import generated.SweepersListGa
 import jetbrains.buildServer.configs.kotlin.Project
 import replaceCharsId
 import vcs_roots.HashiCorpVCSRootGa
@@ -31,7 +31,7 @@ fun projectSweeperSubProject(allConfig: AllContextParameters): Project {
 
     // Create build config for sweeping project resources
     // Uses the HashiCorpVCSRootGa VCS Root so that the latest sweepers in hashicorp/terraform-provider-google are used
-    val serviceSweeperConfig = BuildConfigurationForProjectSweeper("N/A", ProjectSweeperName, SweepersList, projectId, HashiCorpVCSRootGa, sharedResources, gaConfig)
+    val serviceSweeperConfig = BuildConfigurationForProjectSweeper("N/A", ProjectSweeperName, SweepersListGa, projectId, HashiCorpVCSRootGa, sharedResources, gaConfig)
     val trigger  = NightlyTriggerConfiguration(startHour=12)
     serviceSweeperConfig.addTrigger(trigger)
 

mmv1/third_party/terraform/.teamcity/components/projects/reused/mm_upstream.kt

@@ -13,10 +13,12 @@ import ProviderNameGa
 import ServiceSweeperName
 import SharedResourceNameVcr
 import builds.*
-import generated.PackagesList
-import generated.ServicesListGa
+import generated.PackagesListBeta
+import generated.PackagesListGa
 import generated.ServicesListBeta
-import generated.SweepersList
+import generated.ServicesListGa
+import generated.SweepersListBeta
+import generated.SweepersListGa
 import jetbrains.buildServer.configs.kotlin.BuildType
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
@@ -36,7 +38,13 @@ fun mmUpstream(parentProject: String, providerName: String, vcsRoot: GitVcsRoot,
     val packageBuildConfigs = BuildConfigurationsForPackages(allPackages, providerName, projectId, vcsRoot, sharedResources, config)
 
     // Create build config for sweeping the VCR test project - everything except projects
-    val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, SweepersList, projectId, vcsRoot, sharedResources, config)
+    var sweepersList: Map<String,Map<String,String>>
+    when(providerName) {
+        ProviderNameGa -> sweepersList = SweepersListGa
+        ProviderNameBeta -> sweepersList = SweepersListBeta
+        else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
+    }
+    val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config)
     val trigger  = NightlyTriggerConfiguration(startHour=12)
     serviceSweeperConfig.addTrigger(trigger) // Only the sweeper is on a schedule in this project
 
@@ -60,10 +68,10 @@ fun mmUpstream(parentProject: String, providerName: String, vcsRoot: GitVcsRoot,
 fun getAllPackageInProviderVersion(providerName: String): Map<String, Map<String,String>> {
     var allPackages: Map<String, Map<String, String>> = mapOf()
     if (providerName == ProviderNameGa){
-        allPackages = PackagesList + ServicesListGa
+        allPackages = PackagesListGa + ServicesListGa
     }
     if (providerName == ProviderNameBeta){
-        allPackages = PackagesList + ServicesListBeta
+        allPackages = PackagesListBeta + ServicesListBeta
     }
     return allPackages
 }

mmv1/third_party/terraform/.teamcity/components/projects/reused/nightly_tests.kt

@@ -14,8 +14,8 @@ import ServiceSweeperName
 import SharedResourceNameBeta
 import SharedResourceNameGa
 import builds.*
-import generated.PackagesList
-import generated.SweepersList
+import generated.SweepersListBeta
+import generated.SweepersListGa
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
 import replaceCharsId
@@ -44,7 +44,13 @@ fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot
     }
 
     // Create build config for sweeping the nightly test project
-    val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, SweepersList, projectId, vcsRoot, sharedResources, config)
+    var sweepersList: Map<String,Map<String,String>>
+    when(providerName) {
+        ProviderNameGa -> sweepersList = SweepersListGa
+        ProviderNameBeta -> sweepersList = SweepersListBeta
+        else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
+    }
+    val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config)
     val sweeperTrigger  = NightlyTriggerConfiguration(startHour=12)  // Override hour
     serviceSweeperConfig.addTrigger(sweeperTrigger)
 

mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt

@@ -12,7 +12,8 @@ import SharedResourceNameGa
 import SharedResourceNameVcr
 import builds.AllContextParameters
 import builds.readOnlySettings
-import generated.GetPackageNameList
+import generated.PackagesListBeta
+import generated.PackagesListGa
 import generated.ServicesListBeta
 import generated.ServicesListGa
 import jetbrains.buildServer.configs.kotlin.Project
@@ -38,21 +39,21 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
                 id = "GA_NIGHTLY_SERVICE_LOCK_SHARED_RESOURCE"
                 name = SharedResourceNameGa
                 enabled = true
-                resourceType = customValues(getServiceNameList(ServicesListGa) + GetPackageNameList())
+                resourceType = customValues(getPackageNameList(ServicesListGa) + getPackageNameList(PackagesListGa))
             }
             // For controlling sweeping of the Beta nightly test project
             sharedResource {
                 id = "BETA_NIGHTLY_SERVICE_LOCK_SHARED_RESOURCE"
                 name = SharedResourceNameBeta
                 enabled = true
-                resourceType = customValues(getServiceNameList(ServicesListBeta) + GetPackageNameList())
+                resourceType = customValues(getPackageNameList(ServicesListBeta) + getPackageNameList(PackagesListBeta))
             }
             // For controlling sweeping of the PR testing project
             sharedResource {
                 id = "PR_SERVICE_LOCK_SHARED_RESOURCE"
                 name = SharedResourceNameVcr
                 enabled = true
-                resourceType = customValues(getServiceNameList(ServicesListBeta) + GetPackageNameList()) // Use Beta list of services here, assuming Beta is a superset of GA
+                resourceType = customValues(getPackageNameList(ServicesListBeta) + getPackageNameList(PackagesListBeta)) // Use Beta list of services here, assuming Beta is a superset of GA
             }
         }
 
@@ -66,11 +67,11 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
     }
 }
 
-fun getServiceNameList(servicesList: Map<String, Map<String,String>>): List<String> {
+fun getPackageNameList(servicesList: Map<String, Map<String,String>>): List<String> {
     var serviceNameList: ArrayList<String> = arrayListOf()
     servicesList.forEach{ s ->
         var serviceName = s.value.getValue("name").toString()
         serviceNameList.add(serviceName)
     }
     return serviceNameList
-}
+}

mmv1/third_party/terraform/.teamcity/tests/sweepers.kt

@@ -9,6 +9,7 @@ package tests
 
 import ServiceSweeperName
 import jetbrains.buildServer.configs.kotlin.BuildType
+import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import jetbrains.buildServer.configs.kotlin.Project
@@ -62,6 +63,31 @@ class SweeperTests {
         assertTrue("env.SKIP_PROJECT_SWEEPER is set to a non-empty string, so project sweepers are skipped. Value = `${value}` ", value != "")
     }
 
+    @Test
+    fun gaNightlyProjectServiceSweeperRunsInGoogle() {
+        val project = googleCloudRootProject(testContextParameters())
+
+        // Find GA nightly test project
+        val gaProject: Project? =  project.subProjects.find { p->  p.name == gaProjectName}
+        if (gaProject == null) {
+            Assert.fail("Could not find the Google (GA) project")
+        }
+        val gaNightlyTestProject: Project? = gaProject!!.subProjects.find { p->  p.name == nightlyTestsProjectName}
+        if (gaNightlyTestProject == null) {
+            Assert.fail("Could not find the Google (GA) Nightly Test project")
+        }
+
+        // Find sweeper inside
+        val sweeper: BuildType? = gaNightlyTestProject!!.buildTypes.find { p-> p.name == ServiceSweeperName}
+        if (sweeper == null) {
+            Assert.fail("Could not find the sweeper build in the Google (GA) Nightly Test project")
+        }
+
+        // Check PACKAGE_PATH is in google (not google-beta)
+        val value = sweeper!!.params.findRawParam("PACKAGE_PATH")!!.value
+        assertEquals("./google/sweeper", value)
+    }
+
     @Test
     fun betaNightlyProjectServiceSweeperSkipsProjectSweep() {
         val project = googleCloudRootProject(testContextParameters())
@@ -88,4 +114,29 @@ class SweeperTests {
         val value = sweeper!!.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
         assertTrue("env.SKIP_PROJECT_SWEEPER is set to a non-empty string, so project sweepers are skipped. Value = `${value}` ", value != "")
     }
+
+    @Test
+    fun betaNightlyProjectServiceSweeperRunsInGoogleBeta() {
+        val project = googleCloudRootProject(testContextParameters())
+
+        // Find Beta nightly test project
+        val betaProject: Project? =  project.subProjects.find { p->  p.name == betaProjectName}
+        if (betaProject == null) {
+            Assert.fail("Could not find the Google (GA) project")
+        }
+        val betaNightlyTestProject: Project? = betaProject!!.subProjects.find { p->  p.name == nightlyTestsProjectName}
+        if (betaNightlyTestProject == null) {
+            Assert.fail("Could not find the Google (GA) Nightly Test project")
+        }
+
+        // Find sweeper inside
+        val sweeper: BuildType? = betaNightlyTestProject!!.buildTypes.find { p-> p.name == ServiceSweeperName}
+        if (sweeper == null) {
+            Assert.fail("Could not find the sweeper build in the Google (GA) Nightly Test project")
+        }
+
+        // Check PACKAGE_PATH is in google-beta
+        val value = sweeper!!.params.findRawParam("PACKAGE_PATH")!!.value
+        assertEquals("./google-beta/sweeper", value)
+    }
 }