TeamCity: add Ephemeral-write-only subproject (#12538) (#20722)

[upstream:a5601476fba508d1681fd88198e54c82980386c4]

Signed-off-by: Modular Magician <[email protected]>

.changelog/12538.txt

@@ -0,0 +1,3 @@
+```release-note:none
+
+```

.teamcity/components/builds/build_configuration_per_package.kt

@@ -20,7 +20,7 @@ import replaceCharsId
 
 // BuildConfigurationsForPackages accepts a map containing details of multiple packages in a provider and returns a list of build configurations for them all.
 // Intended to be used in projects where we're testing all packages, e.g. the nightly test projects
-fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): List<BuildType> {
+fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): List<BuildType> {
     val list = ArrayList<BuildType>()
 
     // Create build configurations for all packages, except sweeper
@@ -29,7 +29,7 @@ fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, p
         val displayName: String = info.getValue("displayName").toString()
 
         val pkg = PackageDetails(packageName, displayName, providerName, parentProjectName)
-        val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables)
+        val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix)
         list.add(buildConfig)
     }
 
@@ -38,18 +38,17 @@ fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, p
 
 // BuildConfigurationForSinglePackage accepts details of a single package in a provider and returns a build configuration for it
 // Intended to be used in short-lived projects where we're testing specific packages, e.g. feature branch testing
-fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): BuildType{
+fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): BuildType{
     val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName)
-    return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables)
+    return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix)
 }
 
 class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String) {
 
     // buildConfiguration returns a BuildType for a service package
     // For BuildType docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/build-type/index.html
-    fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration): BuildType {
+    fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration, testPrefix: String): BuildType {
 
-        val testPrefix = "TestAcc"
         val testTimeout = "12"
 
         var parallelism = DefaultParallelism

.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
+
+package projects.feature_branches
+
+import ProviderNameBeta
+import ProviderNameGa
+import SharedResourceNameBeta
+import SharedResourceNameGa
+import SharedResourceNameVcr
+import builds.*
+import generated.ServicesListBeta
+import generated.ServicesListGa
+import jetbrains.buildServer.configs.kotlin.Project
+import replaceCharsId
+import vcs_roots.HashiCorpVCSRootBeta
+import vcs_roots.HashiCorpVCSRootGa
+import vcs_roots.ModularMagicianVCSRootBeta
+import vcs_roots.ModularMagicianVCSRootGa
+import components.projects.feature_branches.getServicesList
+import DefaultStartHour
+
+const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only"
+const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211"
+
+fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project {
+
+    val trigger  = NightlyTriggerConfiguration(
+        branch = "refs/heads/$featureBranchEphemeralWriteOnly", // triggered builds must test the feature branch
+        startHour = DefaultStartHour + 6,
+    )
+    val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs
+
+    // GA
+    val gaConfig = getGaAcceptanceTestConfig(allConfig)
+    // These are the packages that have resources that will use write-only attributes
+    var ServicesListWriteOnlyGA = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA")
+
+    val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGA, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig, "TestAcc.*Ephemeral")
+    buildConfigsGa.forEach{ builds ->
+        builds.addTrigger(trigger)
+    }
+
+    var ServicesListWriteOnlyGaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA-MM")
+    val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig, "TestAcc.*Ephemeral")
+
+    // Beta
+    val betaConfig = getBetaAcceptanceTestConfig(allConfig)
+    var ServicesListWriteOnlyBeta = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta")
+    val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig, "TestAcc.*Ephemeral")
+    buildConfigsBeta.forEach{ builds ->
+        builds.addTrigger(trigger)
+    }
+
+    var ServicesListWriteOnlyBetaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta-MM")
+    val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig, "TestAcc.*Ephemeral")
+
+    // Make all builds use a 1.11.0-ish version of TF core
+    val allBuildConfigs = buildConfigsGa + buildConfigsBeta + buildConfigsMMGa + buildConfigsMMBeta
+    allBuildConfigs.forEach{ builds ->
+        builds.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion)
+    }
+
+    // ------
+
+    return Project{
+        id("FEATURE_BRANCH_ephemeral_write_only")
+        name = featureBranchEphemeralWriteOnly
+        description = "Subproject for testing feature branch $featureBranchEphemeralWriteOnly"
+
+        // Register all build configs in the project
+        allBuildConfigs.forEach{ builds ->
+            buildType(builds)
+        }
+
+        params {
+            readOnlySettings()
+        }
+    }
+}

.teamcity/components/projects/feature_branches/get_services.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package components.projects.feature_branches
+
+import generated.ServicesListGa
+import generated.ServicesListBeta
+
+// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
+
+// This function is used to get the services list for a given version. Typically used in feature branch builds for testing very specific services only.
+fun getServicesList(Services: Array<String>, version: String): Map<String,Map<String,String>> {
+    if (Services.isEmpty()) {
+        throw Exception("No services found for version $version")
+    }
+
+    var servicesList = mutableMapOf<String,Map<String,String>>()
+    for (service in Services) {
+        if (version == "GA" || version == "GA-MM") {
+            servicesList[service] = ServicesListGa.getOrElse(service) { throw Exception("Service $service not found") }
+        } else if (version == "Beta" || version == "Beta-MM") {
+            servicesList[service] = ServicesListBeta.getOrElse(service) { throw Exception("Service $service not found") }
+        } else {
+            throw Exception("Invalid version $version")
+        }
+    }
+
+    when (version) {
+        "GA" -> servicesList
+        "Beta" -> {
+            servicesList.mapValues { (_, value) ->
+                value + mapOf(
+                        "displayName" to "${value["displayName"]} - Beta"
+                )
+            }.toMutableMap()
+        }
+        "GA-MM" -> {
+            servicesList.mapValues { (_, value) ->
+                value + mapOf(
+                        "displayName" to "${value["displayName"]} - MM"
+                )
+            }.toMutableMap()
+        }
+        "Beta-MM" -> {
+            servicesList.mapValues { (_, value) ->
+                value + mapOf(
+                        "displayName" to "${value["displayName"]} - Beta - MM"
+                )
+            }.toMutableMap()
+        }
+        else -> throw Exception("Invalid version $version")
+    }.also { servicesList = it as MutableMap<String, Map<String, String>> }
+
+    return servicesList
+}

.teamcity/components/projects/root_project.kt

@@ -18,6 +18,7 @@ import generated.ServicesListBeta
 import generated.ServicesListGa
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.sharedResource
+import projects.feature_branches.featureBranchEphemeralWriteOnlySubProject
 
 // googleCloudRootProject returns a root project that contains a subprojects for the GA and Beta version of the
 // Google provider. There are also resources to help manage the test projects used for acceptance tests.
@@ -61,6 +62,7 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
         subProject(googleSubProjectGa(allConfig))
         subProject(googleSubProjectBeta(allConfig))
         subProject(projectSweeperSubProject(allConfig))
+        subProject(featureBranchEphemeralWriteOnlySubProject(allConfig))
 
         // Feature branch-testing projects - these will be added and removed as needed
 

.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
+
+package tests
+
+import jetbrains.buildServer.configs.kotlin.triggers.ScheduleTrigger
+import org.junit.Assert
+import org.junit.Test
+import projects.feature_branches.featureBranchEphemeralWriteOnly
+import projects.googleCloudRootProject
+
+class FeatureBranchEphemeralWriteOnlySubProject {
+    @Test
+    fun buildsUsingHashiCorpReposAreOnSchedule() {
+        val root = googleCloudRootProject(testContextParameters())
+
+        // Find feature branch project
+        val project = getSubProject(root, featureBranchEphemeralWriteOnly  )
+
+        // All builds using the HashiCorp owned GitHub repos
+        val hashiBuilds = project.buildTypes.filter { bt ->
+            bt.name.contains("HashiCorp downstream")
+        }
+
+        hashiBuilds.forEach{bt ->
+            Assert.assertTrue(
+                "Build configuration `${bt.name}` should contain at least one trigger",
+                bt.triggers.items.isNotEmpty()
+            )
+            // Look for at least one CRON trigger
+            var found = false
+            lateinit var schedulingTrigger: ScheduleTrigger
+            for (item in bt.triggers.items){
+                if (item.type == "schedulingTrigger") {
+                    schedulingTrigger = item as ScheduleTrigger
+                    found = true
+                    break
+                }
+            }
+
+            Assert.assertTrue(
+                "Build configuration `${bt.name}` should contain a CRON/'schedulingTrigger' trigger",
+                found
+            )
+
+            // Check that triggered builds are being run on the feature branch
+            val isCorrectBranch: Boolean = schedulingTrigger.branchFilter == "+:refs/heads/$featureBranchEphemeralWriteOnly"
+
+            Assert.assertTrue(
+                "Build configuration `${bt.name}` is using the $featureBranchEphemeralWriteOnly branch filter",
+                isCorrectBranch
+            )
+        }
+    }
+
+    @Test
+    fun buildsUsingModularMagicianReposAreNotTriggered() {
+        val root = googleCloudRootProject(testContextParameters())
+
+        // Find feature branch project
+        val project = getSubProject(root, featureBranchEphemeralWriteOnly)
+
+        // All builds using the HashiCorp owned GitHub repos
+        val magicianBuilds = project.buildTypes.filter { bt ->
+            bt.name.contains("MM upstream")
+        }
+
+        magicianBuilds.forEach{bt ->
+            Assert.assertTrue(
+                "Build configuration `${bt.name}` should not have any triggers",
+                bt.triggers.items.isEmpty()
+            )
+        }
+    }
+}