Add fixes to Azure

README.md

@@ -1,4 +1,5 @@
 # Security Reference Architectures (SRA) - Terraform Templates
+
 <p align="center">
   <img src="https://i.ibb.co/NrfH2qc/Screenshot-2024-09-17-at-1-02-06-PM.png" />
 </p>
@@ -12,8 +13,93 @@ Security Reference Architecture (SRA) with Terraform templates makes deploying w
 - [Azure](https://github.com/databricks/terraform-databricks-sra/tree/main/azure)
 - [GCP](https://github.com/databricks/terraform-databricks-sra/tree/main/gcp)
 
-## Project support 
+## Project support
 
 Please note the code in this project is provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects. The source in this project is provided subject to the Databricks [License](./LICENSE). All included or referenced third party libraries are subject to the licenses set forth below.
 
-Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support. 
+Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support.
+
+### Example of `dev.tfvars` File
+
+To customize the Terraform configuration for your development environment, create a `dev.tfvars` file with the following content:
+
+```hcl
+# Required Variables
+application_id = "your-application-id"
+databricks_account_id = "your-databricks-account-id"
+location = "your-region"
+
+hub_vnet_cidr = "10.0.0.0/16"
+hub_resource_group_name = "your-hub-resource-group-name"
+hub_vnet_name = "your-hub-vnet-name"
+
+test_vm_password = "your-vm-password"
+client_secret = "your-client-secret"
+databricks_app_object_id = "your-databricks-app-object-id"
+
+# Optional Variables
+public_repos = [
+  "python.org",
+  "*.python.org",
+  "pypi.org",
+  "*.pypi.org",
+  "pythonhosted.org",
+  "*.pythonhosted.org",
+  "cran.r-project.org",
+  "*.cran.r-project.org",
+  "r-project.org"
+]
+
+spoke_config = [
+  {
+    prefix = "spoke1"
+    cidr   = "10.1.0.0/16"
+    tags   = {
+      environment = "dev"
+      owner       = "team1"
+    }
+  },
+  {
+    prefix = "spoke2"
+    cidr   = "10.2.0.0/16"
+    tags   = {
+      environment = "prod"
+      owner       = "team2"
+    }
+  }
+]
+
+tags = {
+  environment = "dev"
+  owner       = "your-team-name"
+}
+```
+
+## Using the Makefile
+
+The provided Makefile simplifies working with Terraform configurations for different platforms and environments. Below is a guide on how to use it.
+
+### Running the Makefile for Different Platforms
+
+1. **Set the `PLATFORM` Variable**
+   Change the `PLATFORM` variable to the desired platform before calling the Makefile. Supported platforms include:
+   - `aws`
+   - `aws-gov`
+   - `azure`
+   - `gcp`
+
+2. **Specify the `ENV` Variable**
+   Set the `ENV` variable to the target environment (e.g., `dev`, `stg`, `prod`).
+
+3. **Terraform Directory and Variables**
+   - The `TERRAFORM_DIR` variable points to the Terraform configuration directory for the selected platform.
+   - The `VARS` variable specifies the path to the `.tfvars` file for the chosen environment.
+
+### Example Command
+
+#### AWS
+```bash
+PLATFORM=aws ENV=dev make plan
+PLATFORM=aws ENV=dev make apply
+PLATFORM=aws ENV=dev make destroy
+```

azure/tf/main.tf

@@ -46,7 +46,14 @@ module "hub" {
   public_repos            = var.public_repos
   test_vm_password        = var.test_vm_password
   client_secret           = var.client_secret
+  application_id          = var.application_id
   tags                    = var.tags
+
+  #options
+  is_kms_enabled           = false
+  is_firewall_enabled      = false
+  is_test_vm_enabled       = false
+  is_unity_catalog_enabled = false
 }
 
 # Define module "spoke" with a for_each loop to iterate over each spoke configuration
@@ -64,7 +71,7 @@ module "spoke" {
 
   location                 = var.location
   route_table_id           = module.hub.route_table_id
-  metastore_id             = module.hub.metastore_id
+  metastore_id             = module.hub.is_unity_catalog_enabled ? module.hub.metastore_id : var.databricks_metastore_id
   hub_vnet_name            = module.hub.vnet_name
   hub_resource_group_name  = module.hub.resource_group_name
   hub_vnet_id              = module.hub.vnet_id
@@ -76,5 +83,11 @@ module "spoke" {
   hub_private_link_info    = module.hub.private_link_info
   tenant_id                = module.hub.tenant_id
 
-  # depends_on = [module.hub]
+  #options
+  is_kms_enabled                      = false
+  is_frontend_private_link_enabled    = false
+  is_storage_private_endpoint_enabled = true
+  boolean_create_private_dbfs         = false
+
+  depends_on = [module.hub]
 }

azure/tf/modules/azure_hub/datasources.tf

@@ -0,0 +1,10 @@
+# Retrieve the current Azure client configuration
+data "azurerm_client_config" "current" {}
+
+# Retrieve the public IP address of the host machine using the ifconfig.co API
+data "http" "my_public_ip" { // add your host machine ip into nsg
+  url = "https://ifconfig.co/json"
+  request_headers = {
+    Accept = "application/json"
+  }
+}

azure/tf/modules/azure_hub/firewall.tf

@@ -1,5 +1,7 @@
 # Define a subnet resource for the Azure Firewall
 resource "azurerm_subnet" "firewall" {
+  count = var.is_firewall_enabled ? 1 : 0
+
   name                 = "AzureFirewallSubnet"
   resource_group_name  = azurerm_resource_group.this.name
   virtual_network_name = azurerm_virtual_network.this.name
@@ -9,11 +11,13 @@ resource "azurerm_subnet" "firewall" {
 
 # Define a public IP resource for the Azure Firewall
 resource "azurerm_public_ip" "this" {
-  name                = "firewall-public-ip"
+  count = var.is_firewall_enabled ? 1 : 0
+
+  name                = "${local.prefix}-fw-public-ip"
   location            = azurerm_resource_group.this.location
   resource_group_name = azurerm_resource_group.this.name
   allocation_method   = "Static"
-  sku                 = "Standard"
+  sku                 = var.firewall_sku
 
   lifecycle {
     ignore_changes = [tags]
@@ -22,31 +26,24 @@ resource "azurerm_public_ip" "this" {
 
 # Define a firewall policy resource
 resource "azurerm_firewall_policy" "this" {
-  name                = "databricks-fwpolicy"
+  count = var.is_firewall_enabled ? 1 : 0
+
+  name                = "${local.prefix}-databricks-fwpolicy"
   resource_group_name = var.hub_resource_group_name
   location            = azurerm_resource_group.this.location
 }
 
-# Define an IP group resource
-resource "azurerm_ip_group" "this" {
-  name                = "databricks-subnets"
-  resource_group_name = azurerm_resource_group.this.name
-  location            = azurerm_resource_group.this.location
-
-  lifecycle {
-    ignore_changes = [cidrs]
-  }
-}
-
 # Define a firewall policy rule collection group resource
 resource "azurerm_firewall_policy_rule_collection_group" "this" {
-  name               = "databricks"
-  firewall_policy_id = azurerm_firewall_policy.this.id
+  count = var.is_firewall_enabled ? 1 : 0
+
+  name               = "${local.prefix}-databricks"
+  firewall_policy_id = azurerm_firewall_policy.this[0].id
   priority           = 200
 
   # Define network rule collection within the rule collection group
   network_rule_collection {
-    name     = "databricks-network-rc"
+    name     = "${local.prefix}-databricks-network-rc"
     priority = 100
     action   = "Allow"
 
@@ -78,7 +75,7 @@ resource "azurerm_firewall_policy_rule_collection_group" "this" {
 
   # Define application rule collection within the rule collection group
   application_rule_collection {
-    name     = "databricks-app-rc"
+    name     = "${local.prefix}-databricks-app-rc"
     priority = 101
     action   = "Allow"
 
@@ -129,18 +126,21 @@ resource "azurerm_firewall_policy_rule_collection_group" "this" {
 
 # Define a firewall resource
 resource "azurerm_firewall" "this" {
+  count = var.is_firewall_enabled ? 1 : 0
+
   name                = "${azurerm_virtual_network.this.name}-firewall"
   location            = azurerm_resource_group.this.location
   resource_group_name = azurerm_resource_group.this.name
   sku_name            = "AZFW_VNet"
-  sku_tier            = "Standard"
-  firewall_policy_id  = azurerm_firewall_policy.this.id
+  sku_tier            = var.firewall_sku
+  firewall_policy_id  = azurerm_firewall_policy.this[0].id
+
 
   # Define IP configuration for the firewall
   ip_configuration {
     name                 = "firewall-public-ip-config"
-    subnet_id            = azurerm_subnet.firewall.id
-    public_ip_address_id = azurerm_public_ip.this.id
+    subnet_id            = azurerm_subnet.firewall[0].id
+    public_ip_address_id = azurerm_public_ip.this[0].id
   }
 
   depends_on = [

azure/tf/modules/azure_hub/keyvault.tf

@@ -1,15 +1,16 @@
 # Why do `key_opts` and `key_permissions` differ in terms of required capitalization?
 # Define the Azure Key Vault resource
-resource "azurerm_key_vault" "example" {
-  name                     = "example-hub-keyvault"
+resource "azurerm_key_vault" "this" {
+  count = var.is_kms_enabled ? 1 : 0
+
+  name                     = "${local.prefix}-kv"
   location                 = azurerm_resource_group.this.location
   resource_group_name      = azurerm_resource_group.this.name
   tenant_id                = local.tenant_id
   purge_protection_enabled = true
   # enable_rbac_authorization = true
 
-  sku_name = "premium"
-
+  sku_name                   = "premium"
   soft_delete_retention_days = 7
 
   lifecycle {
@@ -19,10 +20,10 @@ resource "azurerm_key_vault" "example" {
 
 # Define a key in the Azure Key Vault for managed services
 resource "azurerm_key_vault_key" "managed_services" {
-  depends_on = [azurerm_key_vault_access_policy.terraform]
+  count = var.is_kms_enabled ? 1 : 0
 
-  name         = "adb-services"
-  key_vault_id = azurerm_key_vault.example.id
+  name         = "${local.prefix}-adb-services"
+  key_vault_id = azurerm_key_vault.this[0].id
   key_type     = "RSA"
   key_size     = 2048
 
@@ -35,14 +36,16 @@ resource "azurerm_key_vault_key" "managed_services" {
     "verify",
     "wrapKey",
   ]
+
+  depends_on = [azurerm_key_vault_access_policy.terraform]
 }
 
 # Define a key in the Azure Key Vault for managed disks
 resource "azurerm_key_vault_key" "managed_disk" {
-  depends_on = [azurerm_key_vault_access_policy.terraform]
+  count = var.is_kms_enabled ? 1 : 0
 
-  name         = "adb-disk"
-  key_vault_id = azurerm_key_vault.example.id
+  name         = "${local.prefix}-adb-disk"
+  key_vault_id = azurerm_key_vault.this[0].id
   key_type     = "RSA"
   key_size     = 2048
 
@@ -55,18 +58,16 @@ resource "azurerm_key_vault_key" "managed_disk" {
     "verify",
     "wrapKey",
   ]
-}
 
-# resource "azurerm_role_assignment" "key_vault_reader" {
-#   scope              = azurerm_key_vault.example.id
-#   role_definition_id = "21090545-7ca7-4776-b22c-e363652d74d2"
-#   principal_id       = data.azurerm_client_config.current.object_id
-# }
+  depends_on = [azurerm_key_vault_access_policy.terraform]
+}
 
 # Define an access policy for the Azure Key Vault
 resource "azurerm_key_vault_access_policy" "terraform" {
-  key_vault_id = azurerm_key_vault.example.id
-  tenant_id    = azurerm_key_vault.example.tenant_id
+  count = var.is_kms_enabled ? 1 : 0
+
+  key_vault_id = azurerm_key_vault.this[0].id
+  tenant_id    = azurerm_key_vault.this[0].tenant_id
   object_id    = data.azurerm_client_config.current.object_id
 
   key_permissions = [
@@ -89,33 +90,39 @@ resource "azurerm_key_vault_access_policy" "terraform" {
 }
 
 resource "azurerm_private_dns_zone" "key_vault" {
+  count = var.is_kms_enabled ? 1 : 0
+
   name                = "privatelink.vaultcore.azure.net"
   resource_group_name = azurerm_resource_group.this.name
 }
 
 resource "azurerm_private_endpoint" "key_vault" {
+  count = var.is_kms_enabled ? 1 : 0
+
   name                = "${local.prefix}-kv-pe"
   location            = azurerm_resource_group.this.location
   resource_group_name = azurerm_resource_group.this.name
   subnet_id           = azurerm_subnet.privatelink.id
 
   private_service_connection {
     name                           = "keyvault"
-    private_connection_resource_id = azurerm_key_vault.example.id
+    private_connection_resource_id = azurerm_key_vault.this[0].id
     is_manual_connection           = false
     subresource_names              = ["vault"]
   }
 
   private_dns_zone_group {
     name                 = "keyvault"
-    private_dns_zone_ids = [azurerm_private_dns_zone.key_vault.id]
+    private_dns_zone_ids = [azurerm_private_dns_zone.key_vault[0].id]
   }
 
 }
 
 resource "azurerm_private_dns_zone_virtual_network_link" "key_vault" {
+  count = var.is_kms_enabled ? 1 : 0
+
   name                  = "${local.prefix}-keyvault-vnetlink"
   resource_group_name   = azurerm_resource_group.this.name
-  private_dns_zone_name = azurerm_private_dns_zone.key_vault.name
+  private_dns_zone_name = azurerm_private_dns_zone.key_vault[0].name
   virtual_network_id    = azurerm_virtual_network.this.id
 }