From 1a0b78d88e0ab85131311a925606ce1ee5641bf8 Mon Sep 17 00:00:00 2001 From: The Magician Date: Tue, 17 Dec 2024 11:22:35 -0800 Subject: [PATCH] TeamCity: add Ephemeral-write-only subproject (#12538) (#20722) [upstream:a5601476fba508d1681fd88198e54c82980386c4] Signed-off-by: Modular Magician --- .changelog/12538.txt | 3 + .../builds/build_configuration_per_package.kt | 11 ++- .../FEATURE-BRANCH-ephemeral-write-only.kt | 84 +++++++++++++++++++ .../projects/feature_branches/get_services.kt | 56 +++++++++++++ .teamcity/components/projects/root_project.kt | 2 + .../FEATURE-BRANCH-ephemera-write-only.kt | 79 +++++++++++++++++ 6 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 .changelog/12538.txt create mode 100644 .teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt create mode 100644 .teamcity/components/projects/feature_branches/get_services.kt create mode 100644 .teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt diff --git a/.changelog/12538.txt b/.changelog/12538.txt new file mode 100644 index 00000000000..42b910df155 --- /dev/null +++ b/.changelog/12538.txt @@ -0,0 +1,3 @@ +```release-note:none + +``` \ No newline at end of file diff --git a/.teamcity/components/builds/build_configuration_per_package.kt b/.teamcity/components/builds/build_configuration_per_package.kt index 0e9dcbfc44c..8971a41a037 100644 --- a/.teamcity/components/builds/build_configuration_per_package.kt +++ b/.teamcity/components/builds/build_configuration_per_package.kt @@ -20,7 +20,7 @@ import replaceCharsId // BuildConfigurationsForPackages accepts a map containing details of multiple packages in a provider and returns a list of build configurations for them all. // Intended to be used in projects where we're testing all packages, e.g. the nightly test projects -fun BuildConfigurationsForPackages(packages: Map>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration): List { +fun BuildConfigurationsForPackages(packages: Map>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): List { val list = ArrayList() // Create build configurations for all packages, except sweeper @@ -29,7 +29,7 @@ fun BuildConfigurationsForPackages(packages: Map>, p val displayName: String = info.getValue("displayName").toString() val pkg = PackageDetails(packageName, displayName, providerName, parentProjectName) - val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables) + val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix) list.add(buildConfig) } @@ -38,18 +38,17 @@ fun BuildConfigurationsForPackages(packages: Map>, p // BuildConfigurationForSinglePackage accepts details of a single package in a provider and returns a build configuration for it // Intended to be used in short-lived projects where we're testing specific packages, e.g. feature branch testing -fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration): BuildType{ +fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): BuildType{ val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName) - return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables) + return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix) } class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String) { // buildConfiguration returns a BuildType for a service package // For BuildType docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/build-type/index.html - fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration): BuildType { + fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration, testPrefix: String): BuildType { - val testPrefix = "TestAcc" val testTimeout = "12" var parallelism = DefaultParallelism diff --git a/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt new file mode 100644 index 00000000000..249e8eb517a --- /dev/null +++ b/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -0,0 +1,84 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +package projects.feature_branches + +import ProviderNameBeta +import ProviderNameGa +import SharedResourceNameBeta +import SharedResourceNameGa +import SharedResourceNameVcr +import builds.* +import generated.ServicesListBeta +import generated.ServicesListGa +import jetbrains.buildServer.configs.kotlin.Project +import replaceCharsId +import vcs_roots.HashiCorpVCSRootBeta +import vcs_roots.HashiCorpVCSRootGa +import vcs_roots.ModularMagicianVCSRootBeta +import vcs_roots.ModularMagicianVCSRootGa +import components.projects.feature_branches.getServicesList +import DefaultStartHour + +const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" +const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" + +fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { + + val trigger = NightlyTriggerConfiguration( + branch = "refs/heads/$featureBranchEphemeralWriteOnly", // triggered builds must test the feature branch + startHour = DefaultStartHour + 6, + ) + val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs + + // GA + val gaConfig = getGaAcceptanceTestConfig(allConfig) + // These are the packages that have resources that will use write-only attributes + var ServicesListWriteOnlyGA = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA") + + val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGA, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig, "TestAcc.*Ephemeral") + buildConfigsGa.forEach{ builds -> + builds.addTrigger(trigger) + } + + var ServicesListWriteOnlyGaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA-MM") + val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig, "TestAcc.*Ephemeral") + + // Beta + val betaConfig = getBetaAcceptanceTestConfig(allConfig) + var ServicesListWriteOnlyBeta = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta") + val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig, "TestAcc.*Ephemeral") + buildConfigsBeta.forEach{ builds -> + builds.addTrigger(trigger) + } + + var ServicesListWriteOnlyBetaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta-MM") + val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig, "TestAcc.*Ephemeral") + + // Make all builds use a 1.11.0-ish version of TF core + val allBuildConfigs = buildConfigsGa + buildConfigsBeta + buildConfigsMMGa + buildConfigsMMBeta + allBuildConfigs.forEach{ builds -> + builds.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion) + } + + // ------ + + return Project{ + id("FEATURE_BRANCH_ephemeral_write_only") + name = featureBranchEphemeralWriteOnly + description = "Subproject for testing feature branch $featureBranchEphemeralWriteOnly" + + // Register all build configs in the project + allBuildConfigs.forEach{ builds -> + buildType(builds) + } + + params { + readOnlySettings() + } + } +} \ No newline at end of file diff --git a/.teamcity/components/projects/feature_branches/get_services.kt b/.teamcity/components/projects/feature_branches/get_services.kt new file mode 100644 index 00000000000..14b98a1848f --- /dev/null +++ b/.teamcity/components/projects/feature_branches/get_services.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ +package components.projects.feature_branches + +import generated.ServicesListGa +import generated.ServicesListBeta + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +// This function is used to get the services list for a given version. Typically used in feature branch builds for testing very specific services only. +fun getServicesList(Services: Array, version: String): Map> { + if (Services.isEmpty()) { + throw Exception("No services found for version $version") + } + + var servicesList = mutableMapOf>() + for (service in Services) { + if (version == "GA" || version == "GA-MM") { + servicesList[service] = ServicesListGa.getOrElse(service) { throw Exception("Service $service not found") } + } else if (version == "Beta" || version == "Beta-MM") { + servicesList[service] = ServicesListBeta.getOrElse(service) { throw Exception("Service $service not found") } + } else { + throw Exception("Invalid version $version") + } + } + + when (version) { + "GA" -> servicesList + "Beta" -> { + servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - Beta" + ) + }.toMutableMap() + } + "GA-MM" -> { + servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - MM" + ) + }.toMutableMap() + } + "Beta-MM" -> { + servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - Beta - MM" + ) + }.toMutableMap() + } + else -> throw Exception("Invalid version $version") + }.also { servicesList = it as MutableMap> } + + return servicesList +} \ No newline at end of file diff --git a/.teamcity/components/projects/root_project.kt b/.teamcity/components/projects/root_project.kt index c551f8f20fd..1daac200f2f 100644 --- a/.teamcity/components/projects/root_project.kt +++ b/.teamcity/components/projects/root_project.kt @@ -18,6 +18,7 @@ import generated.ServicesListBeta import generated.ServicesListGa import jetbrains.buildServer.configs.kotlin.Project import jetbrains.buildServer.configs.kotlin.sharedResource +import projects.feature_branches.featureBranchEphemeralWriteOnlySubProject // googleCloudRootProject returns a root project that contains a subprojects for the GA and Beta version of the // Google provider. There are also resources to help manage the test projects used for acceptance tests. @@ -61,6 +62,7 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project { subProject(googleSubProjectGa(allConfig)) subProject(googleSubProjectBeta(allConfig)) subProject(projectSweeperSubProject(allConfig)) + subProject(featureBranchEphemeralWriteOnlySubProject(allConfig)) // Feature branch-testing projects - these will be added and removed as needed diff --git a/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt b/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt new file mode 100644 index 00000000000..643050e0e0d --- /dev/null +++ b/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +package tests + +import jetbrains.buildServer.configs.kotlin.triggers.ScheduleTrigger +import org.junit.Assert +import org.junit.Test +import projects.feature_branches.featureBranchEphemeralWriteOnly +import projects.googleCloudRootProject + +class FeatureBranchEphemeralWriteOnlySubProject { + @Test + fun buildsUsingHashiCorpReposAreOnSchedule() { + val root = googleCloudRootProject(testContextParameters()) + + // Find feature branch project + val project = getSubProject(root, featureBranchEphemeralWriteOnly ) + + // All builds using the HashiCorp owned GitHub repos + val hashiBuilds = project.buildTypes.filter { bt -> + bt.name.contains("HashiCorp downstream") + } + + hashiBuilds.forEach{bt -> + Assert.assertTrue( + "Build configuration `${bt.name}` should contain at least one trigger", + bt.triggers.items.isNotEmpty() + ) + // Look for at least one CRON trigger + var found = false + lateinit var schedulingTrigger: ScheduleTrigger + for (item in bt.triggers.items){ + if (item.type == "schedulingTrigger") { + schedulingTrigger = item as ScheduleTrigger + found = true + break + } + } + + Assert.assertTrue( + "Build configuration `${bt.name}` should contain a CRON/'schedulingTrigger' trigger", + found + ) + + // Check that triggered builds are being run on the feature branch + val isCorrectBranch: Boolean = schedulingTrigger.branchFilter == "+:refs/heads/$featureBranchEphemeralWriteOnly" + + Assert.assertTrue( + "Build configuration `${bt.name}` is using the $featureBranchEphemeralWriteOnly branch filter", + isCorrectBranch + ) + } + } + + @Test + fun buildsUsingModularMagicianReposAreNotTriggered() { + val root = googleCloudRootProject(testContextParameters()) + + // Find feature branch project + val project = getSubProject(root, featureBranchEphemeralWriteOnly) + + // All builds using the HashiCorp owned GitHub repos + val magicianBuilds = project.buildTypes.filter { bt -> + bt.name.contains("MM upstream") + } + + magicianBuilds.forEach{bt -> + Assert.assertTrue( + "Build configuration `${bt.name}` should not have any triggers", + bt.triggers.items.isEmpty() + ) + } + } +}