From 2b9799412d7b5f1923555ab36631534674006534 Mon Sep 17 00:00:00 2001 From: Alex Ott Date: Tue, 17 Dec 2024 10:31:23 -0500 Subject: [PATCH] [Exporter] Correctly handle DB-managed UC objects (#4323) ## Changes Databricks has started to introduce UC objects that are created and managed by Databricks, i.e., storage credentials and external locations, so we can't create resources to manage them. This PR converts such managed UC objects into data sources so we can attach permissions to them and handle references. The change of ownership is still not implemented - filed https://github.com/databricks/terraform-provider-databricks/issues/4321 to handle it. Other changes include: - refactoring of creation of new resource data objects - fixing a problem with the codegen of data sources, so we don't need a dedicated `Body` implementation. ## Tests - [x] `make test` run locally - [ ] relevant change in `docs/` folder - [ ] covered with integration tests in `internal/acceptance` - [ ] using Go SDK - [ ] using TF Plugin Framework --- exporter/codegen.go | 9 +++++++-- exporter/importables.go | 36 +++++++++++++++++++++++++++++------- exporter/util.go | 19 ++++++++++--------- exporter/util_workspace.go | 10 +--------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/exporter/codegen.go b/exporter/codegen.go index 549b836ce4..a49b9fd72c 100644 --- a/exporter/codegen.go +++ b/exporter/codegen.go @@ -662,8 +662,13 @@ func (ic *importContext) processSingleResource(resourcesChan resourceChannel, log.Printf("[ERROR] error calling ir.Body for %v: %s", r, err.Error()) } } else { - resourceBlock := body.AppendNewBlock("resource", []string{r.Resource, r.Name}) - err = ic.dataToHcl(ir, []string{}, ic.Resources[r.Resource], r, resourceBlock.Body()) + blockType := "resource" + if r.Mode == "data" { + blockType = r.Mode + } + resourceBlock := body.AppendNewBlock(blockType, []string{r.Resource, r.Name}) + err = ic.dataToHcl(ic.Importables[r.Resource], + []string{}, ic.Resources[r.Resource], r, resourceBlock.Body()) if err != nil { log.Printf("[ERROR] error generating body for %v: %s", r, err.Error()) } diff --git a/exporter/importables.go b/exporter/importables.go index 058cb86298..0230ef77e2 100644 --- a/exporter/importables.go +++ b/exporter/importables.go @@ -89,7 +89,8 @@ var ( "storage_credential": {`CREATE_EXTERNAL_LOCATION`, `CREATE_EXTERNAL_TABLE`}, "foreign_connection": {`CREATE_FOREIGN_CATALOG`}, } - ParentDirectoryExtraKey = "parent_directory" + ParentDirectoryExtraKey = "parent_directory" + dbManagedExternalLocation = "__databricks_managed_storage_location" ) func generateMountBody(ic *importContext, body *hclwrite.Body, r *resource) error { @@ -899,9 +900,8 @@ var resourcesMap map[string]importable = map[string]importable{ {Path: "libraries.jar", Resource: "databricks_repo", Match: "workspace_path", MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName}, }, - // TODO: special formatting required, where JSON is written line by line - // so that we're able to do the references - Body: resourceOrDataBlockBody, + // TODO: implement a custom Body that will write with special formatting, where + // JSON is written line by line so that we're able to do the references }, "databricks_group": { Service: "groups", @@ -1051,7 +1051,6 @@ var resourcesMap map[string]importable = map[string]importable{ return nil }, - Body: resourceOrDataBlockBody, }, "databricks_group_member": { Service: "groups", @@ -2291,7 +2290,6 @@ var resourcesMap map[string]importable = map[string]importable{ } return nil }, - Body: resourceOrDataBlockBody, Depends: []reference{ {Path: "path", Resource: "databricks_user", Match: "home"}, {Path: "path", Resource: "databricks_service_principal", Match: "home"}, @@ -2964,6 +2962,14 @@ var resourcesMap map[string]importable = map[string]importable{ WorkspaceLevel: true, Service: "uc-storage-credentials", Import: func(ic *importContext, r *resource) error { + if r.ID == "__databricks_managed_storage_credential" { + // it's created by default and can't be imported + // TODO: add check for "securable_kind":"STORAGE_CREDENTIAL_DB_AWS_IAM" when we get it in the credential + r.Mode = "data" + data := tfcatalog.ResourceStorageCredential().ToResource().TestResourceData() + obj := tfcatalog.StorageCredentialInfo{Name: r.ID} + r.Data = ic.generateNewData(data, "databricks_storage_credential", r.ID, obj) + } ic.emitUCGrantsWithOwner("storage_credential/"+r.ID, r) if r.Data != nil { isolationMode := r.Data.Get("isolation_mode").(string) @@ -3036,6 +3042,14 @@ var resourcesMap map[string]importable = map[string]importable{ WorkspaceLevel: true, Service: "uc-external-locations", Import: func(ic *importContext, r *resource) error { + if r.ID == dbManagedExternalLocation { + // it's created by default and can't be imported + // TODO: add check for "securable_kind":"EXTERNAL_LOCATION_DB_STORAGE" when we get it in the credential + r.Mode = "data" + data := tfcatalog.ResourceExternalLocation().ToResource().TestResourceData() + obj := tfcatalog.ExternalLocationInfo{Name: r.ID} + r.Data = ic.generateNewData(data, "databricks_external_location", r.ID, obj) + } ic.emitUCGrantsWithOwner("external_location/"+r.ID, r) credentialName := r.Data.Get("credential_name").(string) ic.Emit(&resource{ @@ -3067,7 +3081,15 @@ var resourcesMap map[string]importable = map[string]importable{ } return nil }, - ShouldOmitField: shouldOmitWithIsolationMode, + ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { + if (pathString == "url" || pathString == "credential_name") && d.Get("name").(string) == dbManagedExternalLocation { + return true + } + if pathString == "isolation_mode" { + return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED" + } + return shouldOmitForUnityCatalog(ic, pathString, as, d) + }, // This external location is automatically created when metastore is created with the `storage_root` Ignore: func(ic *importContext, r *resource) bool { return r.ID == "metastore_default_location" diff --git a/exporter/util.go b/exporter/util.go index ccc5862ca7..3a2b19b920 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/hashicorp/hcl/v2/hclwrite" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) @@ -310,7 +309,7 @@ func (ic *importContext) saveFileIn(dir, name string, content []byte) (string, e return relativeName, nil } -func defaultShouldOmitFieldFunc(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { +func defaultShouldOmitFieldFunc(_ *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { if as.Computed { return true } else if as.Default != nil && d.Get(pathString) == as.Default { @@ -320,14 +319,16 @@ func defaultShouldOmitFieldFunc(ic *importContext, pathString string, as *schema return false } -func resourceOrDataBlockBody(ic *importContext, body *hclwrite.Body, r *resource) error { - blockType := "resource" - if r.Mode == "data" { - blockType = r.Mode +func (ic *importContext) generateNewData(data *schema.ResourceData, resourceType, rID string, obj any) *schema.ResourceData { + data.MarkNewResource() + data.SetId(rID) + scm := ic.Resources[resourceType].Schema + err := common.StructToData(obj, scm, data) + if err != nil { + log.Printf("[ERROR] can't convert %s object to data: %v. obj=%v", resourceType, err, obj) + return nil } - resourceBlock := body.AppendNewBlock(blockType, []string{r.Resource, r.Name}) - return ic.dataToHcl(ic.Importables[r.Resource], - []string{}, ic.Resources[r.Resource], r, resourceBlock.Body()) + return data } func generateUniqueID(v string) string { diff --git a/exporter/util_workspace.go b/exporter/util_workspace.go index f8c0c371f9..b60c9c0fb0 100644 --- a/exporter/util_workspace.go +++ b/exporter/util_workspace.go @@ -9,7 +9,6 @@ import ( "sync/atomic" "time" - "github.com/databricks/terraform-provider-databricks/common" "github.com/databricks/terraform-provider-databricks/workspace" "golang.org/x/exp/slices" @@ -246,14 +245,7 @@ func (ic *importContext) maybeEmitWorkspaceObject(resourceType, path string, obj data = workspace.ResourceDirectory().ToResource().TestResourceData() } if data != nil { - scm := ic.Resources[resourceType].Schema - data.MarkNewResource() - data.SetId(path) - err := common.StructToData(obj, scm, data) - if err != nil { - log.Printf("[ERROR] can't convert %s object to data: %v. obj=%v", resourceType, err, obj) - data = nil - } + data = ic.generateNewData(data, resourceType, path, obj) } } ic.Emit(&resource{