Feature/7 re use new scheduled function

build/int.cloudbuild.yaml

@@ -38,4 +38,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0'

build/lint.cloudbuild.yaml

@@ -21,4 +21,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.6.0'

modules/slo-pipeline/README.md

@@ -63,7 +63,6 @@ See the [fixture project](../../test/setup/main.tf) for an example to create thi
 |------|-------------|
 | exporters | Exporter config |
 | function\_bucket\_name | Cloud Function bucket name |
-| function\_bucket\_object\_name | Cloud Function code GCS object name |
 | function\_name | Cloud Function name |
 | project\_id | Project id |
 | pubsub\_topic\_name | Ingress PubSub topic to SLO pipeline |

modules/slo-pipeline/main.tf

@@ -17,9 +17,7 @@
 locals {
   function_source_directory = var.function_source_directory != "" ? var.function_source_directory : "${path.module}/code"
   bucket_suffix             = random_id.suffix.hex
-
-  # Cloud Function suffix so that it updates on code / config change
-  cf_suffix = "${substr(random_uuid.code_hash.result, 0, 2)}${substr(md5(local_file.exporters.content), 0, 2)}"
+  bucket_name               = "${var.function_bucket_name}-${local.bucket_suffix}"
 }
 
 resource "random_id" "suffix" {
@@ -36,26 +34,6 @@ resource "local_file" "exporters" {
   filename = "${path.module}/code/exporters.json"
 }
 
-# Generate a random uuid that will regenerate when one of the file in the source
-# directory is updated.
-resource "random_uuid" "code_hash" {
-  keepers = {
-    for filename in fileset("${path.module}/code", "**/*") :
-    filename => filemd5("${local.function_source_directory}/${filename}")
-  }
-}
-
-# Regenerate the archive whenever one of the Cloud Function code files, SLO
-# config or Error Budget policy is updated.
-data "archive_file" "main" {
-  type        = "zip"
-  output_path = pathexpand("code-pipeline-${local.cf_suffix}.zip")
-  source_dir  = pathexpand(local.function_source_directory)
-  depends_on = [
-    local_file.exporters
-  ]
-}
-
 resource "google_bigquery_dataset" "main" {
   count                       = length(local.bigquery_configs)
   project                     = local.bigquery_configs[count.index].project_id
@@ -67,35 +45,26 @@ resource "google_bigquery_dataset" "main" {
   default_table_expiration_ms = 525600000 # 1 year
 }
 
-resource "google_storage_bucket" "bucket" {
-  name          = "${var.function_bucket_name}-${local.bucket_suffix}"
-  project       = var.project_id
-  location      = var.storage_bucket_location
-  storage_class = var.storage_bucket_storage_class
-}
+module "event_function" {
+  # TODO update version once event-function is released with new functionality
+  source = "github.com/taylorludwig/terraform-google-event-function?ref=feature%2F37-terraform-created-files-in-archive"
+  # source  = "terraform-google-modules/event-function/google"
+  # version = "~> 1.1"
 
-resource "google_storage_bucket_object" "main" {
-  name                = "code-pipeline-${local.cf_suffix}"
-  bucket              = google_storage_bucket.bucket.name
-  source              = data.archive_file.main.output_path
-  content_disposition = "attachment"
-  content_encoding    = "gzip"
-  content_type        = "application/zip"
-}
+  description            = "SLO Exporter to BigQuery or Stackdriver Monitoring"
+  name                   = var.function_name
+  available_memory_mb    = var.function_memory
+  project_id             = var.project_id
+  region                 = var.region
+  service_account_email  = local.service_account_email
+  source_directory       = local.function_source_directory
+  source_dependent_files = [local_file.exporters]
+  bucket_name            = local.bucket_name
+  runtime                = "python37"
+  timeout_s              = "60"
+  entry_point            = "main"
 
-resource "google_cloudfunctions_function" "function" {
-  description           = "SLO Exporter to BigQuery or Stackdriver Monitoring"
-  name                  = "${var.function_name}-${local.cf_suffix}"
-  available_memory_mb   = var.function_memory
-  project               = var.project_id
-  region                = var.region
-  service_account_email = local.service_account_email
-  source_archive_bucket = google_storage_bucket.bucket.name
-  source_archive_object = google_storage_bucket_object.main.name
-  runtime               = "python37"
-  timeout               = "60"
-  entry_point           = "main"
-  event_trigger {
+  event_trigger = {
     event_type = "providers/cloud.pubsub/eventTypes/topic.publish"
     resource   = "projects/${var.project_id}/topics/${google_pubsub_topic.stream.name}"
   }

modules/slo-pipeline/outputs.tf

@@ -26,17 +26,12 @@ output "exporters" {
 
 output "function_name" {
   description = "Cloud Function name"
-  value       = google_cloudfunctions_function.function.name
+  value       = module.event_function.name
 }
 
 output "function_bucket_name" {
   description = "Cloud Function bucket name"
-  value       = google_storage_bucket.bucket.name
-}
-
-output "function_bucket_object_name" {
-  description = "Cloud Function code GCS object name"
-  value       = google_storage_bucket_object.main.name
+  value       = local.bucket_name
 }
 
 output "pubsub_topic_name" {

modules/slo/main.tf

@@ -18,11 +18,8 @@ locals {
   full_name                 = "slo-${var.config.service_name}-${var.config.feature_name}-${var.config.slo_name}"
   pubsub_configs            = [for e in var.config.exporters : e if lower(e.class) == "pubsub"]
   service_account_email     = var.service_account_email != "" ? var.service_account_email : google_service_account.main[0].email
-  bucket_suffix             = random_id.suffix.hex
   function_source_directory = var.function_source_directory != "" ? var.function_source_directory : "${path.module}/code"
-
-  # Cloud Function suffix so that it updates on code / config change
-  cf_suffix = "${substr(random_uuid.config_hash.result, 0, 2)}${substr(random_uuid.config_hash.result, 0, 2)}"
+  suffix                    = random_id.suffix.hex
 }
 
 resource "random_id" "suffix" {
@@ -39,117 +36,25 @@ resource "local_file" "error_budget_policy" {
   filename = "${path.module}/code/error_budget_policy.json"
 }
 
-# Temporary code to replace module invocation (see bottom of this file) until
-# https://github.com/terraform-google-modules/terraform-google-event-function/issues/37
-# is fixed.
-resource "google_cloud_scheduler_job" "job" {
-  name        = local.full_name
-  project     = var.project_id
-  region      = var.region
-  description = var.config.slo_description
-  schedule    = var.schedule
-  time_zone   = var.time_zone
-
-  pubsub_target {
-    topic_name = "projects/${var.project_id}/topics/${module.pubsub_topic.topic}"
-    data       = var.message_data
-  }
-}
-
-module "pubsub_topic" {
-  source     = "terraform-google-modules/pubsub/google"
-  version    = "~> 1.0"
-  project_id = var.project_id
-  topic      = local.full_name
-}
-
-# Generate a random uuid that will regenerate when one of the file in the source
-# directory is updated.
-resource "random_uuid" "code_hash" {
-  keepers = {
-    for filename in fileset(local.function_source_directory, "**/*") :
-    filename => filemd5("${local.function_source_directory}/${filename}")
-  }
-}
-
-# Generate a random uuid that will regenerate when the SLO config or the Error
-# Budget Policy is updated.
-# Workaround for https://github.com/terraform-providers/terraform-provider-random/issues/95
-resource "random_uuid" "config_hash" {
-  keepers = {
-    for content in [local_file.slo.content, local_file.error_budget_policy.content] :
-    content => md5(content)
-  }
-}
-
-# Regenerate the archive whenever one of the Cloud Function code files, SLO
-# config or Error Budget policy is updated.
-data "archive_file" "main" {
-  type        = "zip"
-  output_path = pathexpand("code-${local.full_name}-${local.cf_suffix}.zip")
-  source_dir  = pathexpand(local.function_source_directory)
-  depends_on = [
-    local_file.slo,
-    local_file.error_budget_policy
-  ]
-}
-
-resource "google_storage_bucket" "main" {
-  name          = "${local.full_name}-${local.bucket_suffix}"
-  force_destroy = var.bucket_force_destroy
-  location      = var.region
-  project       = var.project_id
-  storage_class = "REGIONAL"
-  labels        = var.labels
-}
-
-resource "google_storage_bucket_object" "main" {
-  name                = "code-${local.full_name}-${local.cf_suffix}"
-  bucket              = google_storage_bucket.main.name
-  source              = data.archive_file.main.output_path
-  content_disposition = "attachment"
-  content_encoding    = "gzip"
-  content_type        = "application/zip"
+module "slo_cloud_function" {
+  # TODO update version once event-function is released with new functionality
+  # source                              = "github.com/terraform-google-modules/terraform-google-scheduled-function"
+  # version                             = "~> 1.1"
+  source                                = "github.com/taylorludwig/terraform-google-scheduled-function?ref=feature%2Fuse-new-event-function"
+  project_id                            = var.project_id
+  region                                = var.region
+  job_schedule                          = var.schedule
+  job_name                              = local.full_name
+  topic_name                            = local.full_name
+  bucket_name                           = "${local.full_name}-${local.suffix}"
+  function_name                         = "${local.full_name}-${local.suffix}"
+  function_description                  = var.config.slo_description
+  function_entry_point                  = "main"
+  function_source_directory             = local.function_source_directory
+  function_source_dependent_files       = [local_file.error_budget_policy, local_file.slo]
+  function_available_memory_mb          = 128
+  function_runtime                      = "python37"
+  function_source_archive_bucket_labels = var.labels
+  function_service_account_email        = local.service_account_email
+  function_labels                       = var.labels
 }
-
-resource "google_cloudfunctions_function" "main" {
-  name                  = "${local.full_name}-${local.cf_suffix}"
-  description           = var.config.slo_description
-  available_memory_mb   = 128
-  timeout               = var.function_timeout_s
-  entry_point           = "main"
-  labels                = var.labels
-  runtime               = "python37"
-  environment_variables = var.function_environment_variables
-  source_archive_bucket = google_storage_bucket.main.name
-  source_archive_object = google_storage_bucket_object.main.name
-  project               = var.project_id
-  region                = var.region
-  service_account_email = local.service_account_email
-
-  event_trigger {
-    event_type = "google.pubsub.topic.publish"
-    resource   = module.pubsub_topic.topic
-  }
-}
-
-# Replace below code by this once https://github.com/terraform-google-modules/terraform-google-event-function/issues/37
-# is fixed.
-# module "slo-cloud-function" {
-#   source                                = "github.com/terraform-google-modules/terraform-google-scheduled-function"
-#   project_id                            = var.project_id
-#   region                                = var.region
-#   job_schedule                          = var.schedule
-#   job_name                              = local.full_name
-#   topic_name                            = local.full_name
-#   bucket_name                           = "${local.full_name}-${local.bucket_suffix}"
-#   function_name                         = "${local.full_name}-${local.cf_suffix}"
-#   function_description                  = var.config.slo_description
-#   function_entry_point                  = "main"
-#   function_source_directory             = "${path.module}/code"
-#   function_available_memory_mb          = 128
-#   function_runtime                      = "python37"
-#   function_source_archive_bucket_labels = var.labels
-#   function_service_account_email        = local.service_account_email
-#   function_labels                       = var.labels
-# }

modules/slo/outputs.tf

@@ -31,10 +31,10 @@ output "service_account_email" {
 
 output "scheduler_job_name" {
   description = "Cloud Scheduler job name"
-  value       = google_cloud_scheduler_job.job.name
+  value       = module.slo_cloud_function.name
 }
 
 output "function_zip_output_path" {
   description = "Cloud Function zip output path"
-  value       = data.archive_file.main.output_path
+  value       = "${local.function_source_directory}.zip"
 }

test/integration/simple_example/controls/gcp.rb

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+slo_pipeline = attribute('slo_pipeline')
+
 control "gcp" do
   title "GCP Resources"
 
-  describe google_storage_bucket(name: attribute("bucket_name")) do
+  describe google_storage_bucket(name: slo_pipeline["function_bucket_name"]) do
     it { should exist }
   end
 end

test/integration/simple_example/controls/gsutil.rb

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+slo_pipeline = attribute('slo_pipeline')
+
 control "gsutil" do
   title "gsutil"
 
   describe command("gsutil ls -p #{attribute("project_id")}") do
     its(:exit_status) { should eq 0 }
     its(:stderr) { should eq "" }
-    its(:stdout) { should match "gs://#{attribute("bucket_name")}" }
+    its(:stdout) { should match "gs://#{slo_pipeline["function_bucket_name"]}" }
   end
 end

test/integration/simple_example/inspec.yml

@@ -34,3 +34,7 @@ attributes:
   - name: labels
     required: false
     type: string
+
+  - name: slo_pipeline
+    required: true
+    type: hash

test/setup/main.tf

@@ -32,7 +32,9 @@ module "project" {
     "logging.googleapis.com",
     "monitoring.googleapis.com",
     "pubsub.googleapis.com",
-    "storage.googleapis.com"
+    "iam.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
+    "serviceusage.googleapis.com"
   ]
 }
 

test/setup/outputs.tf

@@ -18,6 +18,10 @@ output "project_id" {
   value = module.project.project_id
 }
 
+output "stackdriver_host_project_id" {
+  value = module.project.project_id
+}
+
 output "sa_key" {
   value     = google_service_account_key.int_test.private_key
   sensitive = true

test/setup/versions.tf

@@ -19,9 +19,9 @@ terraform {
 }
 
 provider "google" {
-  version = "~> 2.13.0"
+  version = "~> 2.12.0"
 }
 
 provider "google-beta" {
-  version = "~> 2.13.0"
+  version = "~> 2.12.0"
 }