Using root ca when connect to storage (#193)

* Support of storage tls connection has been added

deploy/ydb-operator/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.3
+version: 0.5.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.5.3"
+appVersion: "0.5.4"

e2e/tests/data/ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIRAKt07W/6Gy+wIe5lk+0YyqwwDQYJKoZIhvcNAQELBQAw
+FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMCAXDTI0MDQxMjA4MTM0NFoYDzIwNTQw
+NDA1MDgxMzQ0WjAXMRUwEwYDVQQDEwx0ZXN0LXJvb3QtY2EwggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQCs4c67HN45wf9jokQLdxDsfLUO5I3FiPVE4uWZ
+Ma2zNSL2pMJBk95Vmj6pP/3HA6llUm3flVotzVzHh3C0j/WBZf6YE31eWlyMokuE
+uLAGfKw/qL+gqC6Phoa72f9kJwnGXsVMDZijAEyqNquLZwgkK+4jgQcVhpGi/3ws
+fop0qYVcK5LKAT5lGSx0MEuW74jheLDlscMsmUqVl2SCWRC/UGY+nUOTpcKK8228
+Corc+DEFstqOIXGH9n/k0ZmBxjh8eU4IRp+LiDcB6x/yI4edAYJK/mnejmSA2i6a
+K2mSzCJfBSVnDxwiGWY6xm8eAh6MaDU6iuqqkFFPltGl90CRAgMBAAGjQjBAMA4G
+A1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTFXnzlk4tO
+SosBlUEM7AiDOYuW7TANBgkqhkiG9w0BAQsFAAOCAQEAJHTeKc1ySltDwwINFVp1
+z5kFlIMyp3l146xn6qT5VWzYP4dZWdJz3gjAML56HRCHNe6B3MijjQY8sRObD5YI
+589xpEhLMr+JR/DmU3Yol0XGILUdZ6TeK6FK+U3gYJdy3U39rcV2usEGfN5SRV4b
+rUZg8asLFWPY7cdWBNIkF2yuJcF6PIpnuhzbfiEtOZ9ucvgnc62XPDnIuMSBKojG
+Bj7QfqadEddSOztZFL00FZULIwSVE/8o0+HQvTBGjbZMuvuSBTJswujYUkD5Sy2O
+wfc+dMJ6dbcDZR+5Q1kLEg+Oq7jjx/BTS35Axo1fPxO/WumJppTUmuEbFtEoX47o
+fQ==
+-----END CERTIFICATE-----

e2e/tests/data/storage-block-4-2-config-tls.yaml

@@ -0,0 +1,115 @@
+static_erasure: block-4-2
+host_configs:
+  - drive:
+      - path: SectorMap:1:1
+        type: SSD
+    host_config_id: 1
+domains_config:
+  domain:
+    - name: Root
+      storage_pool_types:
+        - kind: ssd
+          pool_config:
+            box_id: 1
+            erasure_species: block-4-2
+            kind: ssd
+            pdisk_filter:
+              - property:
+                  - type: SSD
+            vdisk_kind: Default
+  state_storage:
+    - ring:
+        node: [1, 2, 3, 4, 5, 6, 7, 8]
+        nto_select: 5
+      ssid: 1
+table_service_config:
+  sql_version: 1
+actor_system_config:
+  executor:
+    - name: System
+      threads: 1
+      type: BASIC
+    - name: User
+      threads: 1
+      type: BASIC
+    - name: Batch
+      threads: 1
+      type: BASIC
+    - name: IO
+      threads: 1
+      time_per_mailbox_micro_secs: 100
+      type: IO
+    - name: IC
+      spin_threshold: 10
+      threads: 4
+      time_per_mailbox_micro_secs: 100
+      type: BASIC
+  scheduler:
+    progress_threshold: 10000
+    resolution: 256
+    spin_threshold: 0
+blob_storage_config:
+  service_set:
+    groups:
+      - erasure_species: block-4-2
+        rings:
+          - fail_domains:
+              - vdisk_locations:
+                  - node_id: storage-0
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-1
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-2
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-3
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-4
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-5
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-6
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+              - vdisk_locations:
+                  - node_id: storage-7
+                    pdisk_category: SSD
+                    path: SectorMap:1:1
+channel_profile_config:
+  profile:
+    - channel:
+        - erasure_species: block-4-2
+          pdisk_category: 1
+          storage_pool_kind: ssd
+        - erasure_species: block-4-2
+          pdisk_category: 1
+          storage_pool_kind: ssd
+        - erasure_species: block-4-2
+          pdisk_category: 1
+          storage_pool_kind: ssd
+      profile_id: 0
+grpc_config:
+  start_grpc_proxy: true
+  ssl_port: 2135
+  ca: /tls/grpc/ca.crt
+  cert: /tls/grpc/tls.crt
+  key: /tls/grpc/tls.key
+  grpc_memory_quota_bytes: '1073741824'
+  host: '[::]'
+  keep_alive_enable: true
+  keep_alive_idle_timeout_trigger_sec: 90
+  keep_alive_max_probe_count: 3
+  keep_alive_probe_interval_sec: 10
+  services: [legacy, yql, scripting, cms, discovery, monitoring, import, export, locking, maintenance]
+  streaming_config: {enable_output_streams: true}

e2e/tests/data/tls.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDYDCCAkigAwIBAgIRAKHQc3G24VDsWff5qQxyl3EwDQYJKoZIhvcNAQELBQAw
+FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMB4XDTI0MDQxMjA4MTgxOVoXDTM5MDQw
+OTA4MTgxOVowDzENMAsGA1UEChMEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMqGcyqm3UIEZCL3Ge8Qh63OU0QykI3ymtj9gDJ57vtZdUOpHF18
+QsX4Urf6vGNvI8gWXEtPbK8Dchqo+3Reuejjq8aRFs1RaPVZoWPD8i782p6L6/oX
+4k5zAwvCdjC2y/YuUf4GTZqpfwDbhSfH+EdacrfwmYjLxYaehEn7z9M67R5wSekr
+1bMuxBKZW1sclmip3JRf3uBuHfMLkoYpTa7KcEycn9YstZ1h2XuWFLk42GS4bMZs
+Cq2E3sEqD8LOyCHs/qWWGM3Txz6edLTi5U6XtzPeWwxyM5W7fOpC0Q3iw58myGsm
+DDmUngyneS/mzyLvKTywzsu2sRAqdjqsZCUCAwEAAaOBrjCBqzAdBgNVHSUEFjAU
+BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBTF
+Xnzlk4tOSosBlUEM7AiDOYuW7TBbBgNVHREEVDBSgiJzdG9yYWdlLWdycGMueWRi
+LnN2Yy5jbHVzdGVyLmxvY2FsgiwqLnN0b3JhZ2UtaW50ZXJjb25uZWN0LnlkYi5z
+dmMuY2x1c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEAQYRvaGGFsnR4cp8Y
+MJo948t7zI3Pgy20YonmYTriz1zeEYNj9+5t678p04FlCjIx0j4dad1tFC1bNtnI
+FOJNMkhiyC1JPKSN7HR90L5P9JfsuunUVTNEHP6EuLj2/VnMXj+30qX9i0kVkcnH
+OV9yXk8cpN0GuUUyfKUq5k/WS+/15JRoLJ9F8vS1lm77nHeQ9F7m1Yjqkc131N3k
+czFLe3wexWhnKCnNw5OtdFgJsMgL09pYgnN+0xyE4iLY8f0o/b58J/tpbKkeJb9L
+YogP1+ultfGvrlX8TE6iAEi8UA+rZCNpGwiqTI2Y9clZNnkYu7UgW7kn2ih5/vgU
+xntdsA==
+-----END CERTIFICATE-----

e2e/tests/data/tls.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAyoZzKqbdQgRkIvcZ7xCHrc5TRDKQjfKa2P2AMnnu+1l1Q6kc
+XXxCxfhSt/q8Y28jyBZcS09srwNyGqj7dF656OOrxpEWzVFo9VmhY8PyLvzanovr
++hfiTnMDC8J2MLbL9i5R/gZNmql/ANuFJ8f4R1pyt/CZiMvFhp6ESfvP0zrtHnBJ
+6SvVsy7EEplbWxyWaKnclF/e4G4d8wuShilNrspwTJyf1iy1nWHZe5YUuTjYZLhs
+xmwKrYTewSoPws7IIez+pZYYzdPHPp50tOLlTpe3M95bDHIzlbt86kLRDeLDnybI
+ayYMOZSeDKd5L+bPIu8pPLDOy7axECp2OqxkJQIDAQABAoIBAQCXKyBPl9nTax+r
+kbID5dzAeR9h6jRIH+xBR4cnJiih6LZE2LfZd+UHjEGCHl/8AHs+4KHnfNNtFy9W
+gweeZw5xrW8MekQA4WFssYhrxVjChe5RJbPwK1+6mtKNNout9OPtT8nXyLCoXxfz
+defAN900tWinr6mKmD9KKowoBROtYBsKVwO4DOl+JonZP7A5eF/ao4T8XFSOOju5
+aU9flCjw6iagzKuryrJCzt4LsTA3/svuMswPQ7LRMSJm+RsceiBl4oVCuRmJ79tN
+pmhHZqG80upp/idgyuGZu+rgePiXSnmtn47iXXvCjZKTNMkabQMc2WZ5P3HWfG0/
+28kXH5IBAoGBAPswl0SrvZdcQbalNHjLc1JtMMhZs6LPZe+/QTP4i/5qXvee8Dox
+vvGJlvjNQuOGNsPUTsLMY203eKQWJ4+soK2qDhuDCxGmyQJ7NgTTzdhOCULctXlD
+pdBHxiSmCUGEtW7Goe3M1/CL3sVfRp85FzMYItHHZEMTHZNzoZ+XXAzBAoGBAM5n
+SknfqHM4oBT0LxxkmVFyiFoCi2grXzsGjzK5rk30DWajwT5sAy9x+X2vmG2zKkBJ
+tTBYYUFh+SlNtBsY/TW2ZIcmHDpu0w82zXXDLLhUwYjxudXzyG30yU8d4woAwE6u
+NPiVmFX6C8U597a/nC36LLsrSnUPvaJ4voFyRlxlAoGAHJa8MLmnO2nppMMKxNDL
+EE+TJMpo0pfuTyoiXqrkLBGpO1+gkc8Fn3H8d9bMzR6CbyljyXH/wvd0SKCo4gZQ
+x1M6hdEVWm30JM8nJ8d/fyXqkeySzvlvDtSMbbFkDkvvZms/FNSioyMYOLiOTiLu
+TAdsNxoNhEDRte2MMKDGfkECgYAGH1o8xr2gbVWSSYv8M5+4osUYpmqsNF0myxME
+Vi2tckfTe5gH2fxeM+tKpyLGXkIqlgUh4f1Aiz9w0jU9eIhKR5bDy4Wa1h68nMuL
+araw4RK8lS8GAa04VcKC7kgFy+/oZZJ8rTNPmZMvzoBik1x2oK0jAC29OzJM13gP
+LuyXYQKBgQCvkar/nthVUlScVa3uPzqKdpNlqmeXQRMVwcmRQJcJoHYWkaF2RRAx
+AcjWTkP6/ThUgPhI76O0neQqO2N0P+FnCkaE4qU6Wg+dZcH0q7HVnt2TRuCzsECQ
+kxDqdbF+qbXmYfbJ4dKkJdF0dNad4/d2hL/wbvyHXDEEGpuIlZAZrg==
+-----END RSA PRIVATE KEY-----

e2e/tests/smoke_test.go

@@ -551,6 +551,57 @@ var _ = Describe("Operator smoke test", func() {
 		}
 	})
 
+	It("using grpcs for storage connection", func() {
+		By("create secret...")
+		cert := testobjects.DefaultCertificate(
+			filepath.Join(".", "data", "tls.crt"),
+			filepath.Join(".", "data", "tls.key"),
+			filepath.Join(".", "data", "ca.crt"),
+		)
+		Expect(k8sClient.Create(ctx, cert)).Should(Succeed())
+		defer func() {
+			Expect(k8sClient.Delete(ctx, cert)).Should(Succeed())
+		}()
+
+		By("create storage...")
+		storageSample = testobjects.DefaultStorage(filepath.Join(".", "data", "storage-block-4-2-config-tls.yaml"))
+		storageSample.Spec.Service.GRPC.TLSConfiguration.Enabled = true
+		storageSample.Spec.Service.GRPC.TLSConfiguration.Certificate = corev1.SecretKeySelector{
+			LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
+			Key:                  "tls.crt",
+		}
+		storageSample.Spec.Service.GRPC.TLSConfiguration.Key = corev1.SecretKeySelector{
+			LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
+			Key:                  "tls.key",
+		}
+		storageSample.Spec.Service.GRPC.TLSConfiguration.CertificateAuthority = corev1.SecretKeySelector{
+			LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
+			Key:                  "ca.crt",
+		}
+
+		Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed())
+		defer func() {
+			Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed())
+		}()
+		By("create database...")
+		Expect(k8sClient.Create(ctx, databaseSample)).Should(Succeed())
+		defer func() {
+			Expect(k8sClient.Delete(ctx, databaseSample)).Should(Succeed())
+		}()
+
+		By("waiting until Storage is ready...")
+		waitUntilStorageReady(ctx, storageSample.Name, testobjects.YdbNamespace)
+
+		By("checking that all the storage pods are running and ready...")
+		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes)
+
+		By("waiting until database is ready...")
+		waitUntilDatabaseReady(ctx, databaseSample.Name, testobjects.YdbNamespace)
+
+		By("checking that all the database pods are running and ready...")
+		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
+	})
+
 	AfterEach(func() {
 		Expect(uninstallOperatorWithHelm(testobjects.YdbNamespace)).Should(BeTrue())
 		Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())

e2e/tests/test-objects/objects.go

@@ -11,12 +11,13 @@ import (
 )
 
 const (
-	YdbImage      = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17"
-	YdbNamespace  = "ydb"
-	StorageName   = "storage"
-	DatabaseName  = "database"
-	DefaultDomain = "Root"
-	ReadyStatus   = "Ready"
+	YdbImage              = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17"
+	YdbNamespace          = "ydb"
+	StorageName           = "storage"
+	DatabaseName          = "database"
+	CertificateSecretName = "storage-crt"
+	DefaultDomain         = "Root"
+	ReadyStatus           = "Ready"
 )
 
 func constructAntiAffinityFor(key, value string) *corev1.Affinity {
@@ -162,3 +163,25 @@ func DefaultDatabase() *v1alpha1.Database {
 		},
 	}
 }
+
+func DefaultCertificate(certPath, keyPath, caPath string) *corev1.Secret {
+	cert, err := os.ReadFile(certPath)
+	Expect(err).To(BeNil())
+	key, err := os.ReadFile(keyPath)
+	Expect(err).To(BeNil())
+	ca, err := os.ReadFile(caPath)
+	Expect(err).To(BeNil())
+
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      CertificateSecretName,
+			Namespace: YdbNamespace,
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"ca.crt":  ca,
+			"tls.crt": cert,
+			"tls.key": key,
+		},
+	}
+}

internal/cms/tenant.go

@@ -31,6 +31,7 @@ func (t *Tenant) Create(
 	ctx context.Context,
 	database *resources.DatabaseBuilder,
 	creds ydbCredentials.Credentials,
+	opts ...ydb.Option,
 ) error {
 	logger := log.FromContext(ctx)
 	createDatabaseURL := fmt.Sprintf(
@@ -42,6 +43,7 @@ func (t *Tenant) Create(
 	db, err := connection.Open(ctx,
 		createDatabaseURL,
 		ydb.WithCredentials(creds),
+		ydb.MergeOptions(opts...),
 	)
 	if err != nil {
 		logger.Error(err, "Error connecting to YDB storage")

internal/connection/connection.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"time"
 
@@ -41,14 +42,26 @@ func Close(ctx context.Context, db *ydb.Driver) {
 	}
 }
 
-func LoadTLSCredentials(secure bool) grpc.DialOption {
-	if secure {
-		certPool, _ := x509.SystemCertPool()
-		tlsConfig := &tls.Config{
-			MinVersion: tls.VersionTLS12,
-			RootCAs:    certPool,
+func LoadTLSCredentials(secure bool, caBundle []byte) (grpc.DialOption, error) {
+	if !secure {
+		return grpc.WithTransportCredentials(insecure.NewCredentials()), nil
+	}
+	var certPool *x509.CertPool
+	if len(caBundle) > 0 {
+		certPool = x509.NewCertPool()
+		if ok := certPool.AppendCertsFromPEM(caBundle); !ok {
+			return nil, errors.New("failed to parse CA bundle")
+		}
+	} else {
+		var err error
+		certPool, err = x509.SystemCertPool()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get system cert pool, error: %w", err)
 		}
-		return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
 	}
-	return grpc.WithTransportCredentials(insecure.NewCredentials())
+	tlsConfig := &tls.Config{
+		MinVersion: tls.VersionTLS12,
+		RootCAs:    certPool,
+	}
+	return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), nil
 }

internal/controllers/database/init.go

@@ -178,8 +178,18 @@ func (r *Reconciler) handleTenantCreation(
 		)
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
+	tlsOptions, err := resources.GetYDBTLSOption(ctx, database.Storage, r.Config)
+	if err != nil {
+		r.Recorder.Event(
+			database,
+			corev1.EventTypeWarning,
+			"ControllerError",
+			fmt.Sprintf("Failed to get YDB TLS options: %s", err),
+		)
+		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
+	}
 
-	err = tenant.Create(ctx, database, creds)
+	err = tenant.Create(ctx, database, creds, tlsOptions)
 	if err != nil {
 		r.Recorder.Event(
 			database,