From b54e0b91a2d3ee72908aee3102218075948b3b23 Mon Sep 17 00:00:00 2001 From: Mikhail Babich Date: Wed, 17 Apr 2024 11:07:27 +0200 Subject: [PATCH] Using root ca when connect to storage (#193) * Support of storage tls connection has been added --- deploy/ydb-operator/Chart.yaml | 4 +- e2e/tests/data/ca.crt | 19 +++ .../data/storage-block-4-2-config-tls.yaml | 115 ++++++++++++++++++ e2e/tests/data/tls.crt | 21 ++++ e2e/tests/data/tls.key | 27 ++++ e2e/tests/smoke_test.go | 51 ++++++++ e2e/tests/test-objects/objects.go | 35 +++++- internal/cms/tenant.go | 2 + internal/connection/connection.go | 29 +++-- internal/controllers/database/init.go | 12 +- internal/controllers/storage/sync.go | 12 +- internal/healthcheck/healthcheck.go | 8 +- internal/resources/resource.go | 105 ++++++++++++---- 13 files changed, 400 insertions(+), 40 deletions(-) create mode 100644 e2e/tests/data/ca.crt create mode 100644 e2e/tests/data/storage-block-4-2-config-tls.yaml create mode 100644 e2e/tests/data/tls.crt create mode 100644 e2e/tests/data/tls.key diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index c4a7109d..d74fd35a 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.3 +version: 0.5.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.5.3" +appVersion: "0.5.4" diff --git a/e2e/tests/data/ca.crt b/e2e/tests/data/ca.crt new file mode 100644 index 00000000..2ef75164 --- /dev/null +++ b/e2e/tests/data/ca.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIRAKt07W/6Gy+wIe5lk+0YyqwwDQYJKoZIhvcNAQELBQAw +FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMCAXDTI0MDQxMjA4MTM0NFoYDzIwNTQw +NDA1MDgxMzQ0WjAXMRUwEwYDVQQDEwx0ZXN0LXJvb3QtY2EwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCs4c67HN45wf9jokQLdxDsfLUO5I3FiPVE4uWZ +Ma2zNSL2pMJBk95Vmj6pP/3HA6llUm3flVotzVzHh3C0j/WBZf6YE31eWlyMokuE +uLAGfKw/qL+gqC6Phoa72f9kJwnGXsVMDZijAEyqNquLZwgkK+4jgQcVhpGi/3ws +fop0qYVcK5LKAT5lGSx0MEuW74jheLDlscMsmUqVl2SCWRC/UGY+nUOTpcKK8228 +Corc+DEFstqOIXGH9n/k0ZmBxjh8eU4IRp+LiDcB6x/yI4edAYJK/mnejmSA2i6a +K2mSzCJfBSVnDxwiGWY6xm8eAh6MaDU6iuqqkFFPltGl90CRAgMBAAGjQjBAMA4G +A1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTFXnzlk4tO +SosBlUEM7AiDOYuW7TANBgkqhkiG9w0BAQsFAAOCAQEAJHTeKc1ySltDwwINFVp1 +z5kFlIMyp3l146xn6qT5VWzYP4dZWdJz3gjAML56HRCHNe6B3MijjQY8sRObD5YI +589xpEhLMr+JR/DmU3Yol0XGILUdZ6TeK6FK+U3gYJdy3U39rcV2usEGfN5SRV4b +rUZg8asLFWPY7cdWBNIkF2yuJcF6PIpnuhzbfiEtOZ9ucvgnc62XPDnIuMSBKojG +Bj7QfqadEddSOztZFL00FZULIwSVE/8o0+HQvTBGjbZMuvuSBTJswujYUkD5Sy2O +wfc+dMJ6dbcDZR+5Q1kLEg+Oq7jjx/BTS35Axo1fPxO/WumJppTUmuEbFtEoX47o +fQ== +-----END CERTIFICATE----- diff --git a/e2e/tests/data/storage-block-4-2-config-tls.yaml b/e2e/tests/data/storage-block-4-2-config-tls.yaml new file mode 100644 index 00000000..33948d8d --- /dev/null +++ b/e2e/tests/data/storage-block-4-2-config-tls.yaml @@ -0,0 +1,115 @@ +static_erasure: block-4-2 +host_configs: + - drive: + - path: SectorMap:1:1 + type: SSD + host_config_id: 1 +domains_config: + domain: + - name: Root + storage_pool_types: + - kind: ssd + pool_config: + box_id: 1 + erasure_species: block-4-2 + kind: ssd + pdisk_filter: + - property: + - type: SSD + vdisk_kind: Default + state_storage: + - ring: + node: [1, 2, 3, 4, 5, 6, 7, 8] + nto_select: 5 + ssid: 1 +table_service_config: + sql_version: 1 +actor_system_config: + executor: + - name: System + threads: 1 + type: BASIC + - name: User + threads: 1 + type: BASIC + - name: Batch + threads: 1 + type: BASIC + - name: IO + threads: 1 + time_per_mailbox_micro_secs: 100 + type: IO + - name: IC + spin_threshold: 10 + threads: 4 + time_per_mailbox_micro_secs: 100 + type: BASIC + scheduler: + progress_threshold: 10000 + resolution: 256 + spin_threshold: 0 +blob_storage_config: + service_set: + groups: + - erasure_species: block-4-2 + rings: + - fail_domains: + - vdisk_locations: + - node_id: storage-0 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-1 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-2 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-3 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-4 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-5 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-6 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-7 + pdisk_category: SSD + path: SectorMap:1:1 +channel_profile_config: + profile: + - channel: + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + profile_id: 0 +grpc_config: + start_grpc_proxy: true + ssl_port: 2135 + ca: /tls/grpc/ca.crt + cert: /tls/grpc/tls.crt + key: /tls/grpc/tls.key + grpc_memory_quota_bytes: '1073741824' + host: '[::]' + keep_alive_enable: true + keep_alive_idle_timeout_trigger_sec: 90 + keep_alive_max_probe_count: 3 + keep_alive_probe_interval_sec: 10 + services: [legacy, yql, scripting, cms, discovery, monitoring, import, export, locking, maintenance] + streaming_config: {enable_output_streams: true} diff --git a/e2e/tests/data/tls.crt b/e2e/tests/data/tls.crt new file mode 100644 index 00000000..9549d538 --- /dev/null +++ b/e2e/tests/data/tls.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDYDCCAkigAwIBAgIRAKHQc3G24VDsWff5qQxyl3EwDQYJKoZIhvcNAQELBQAw +FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMB4XDTI0MDQxMjA4MTgxOVoXDTM5MDQw +OTA4MTgxOVowDzENMAsGA1UEChMEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMqGcyqm3UIEZCL3Ge8Qh63OU0QykI3ymtj9gDJ57vtZdUOpHF18 +QsX4Urf6vGNvI8gWXEtPbK8Dchqo+3Reuejjq8aRFs1RaPVZoWPD8i782p6L6/oX +4k5zAwvCdjC2y/YuUf4GTZqpfwDbhSfH+EdacrfwmYjLxYaehEn7z9M67R5wSekr +1bMuxBKZW1sclmip3JRf3uBuHfMLkoYpTa7KcEycn9YstZ1h2XuWFLk42GS4bMZs +Cq2E3sEqD8LOyCHs/qWWGM3Txz6edLTi5U6XtzPeWwxyM5W7fOpC0Q3iw58myGsm +DDmUngyneS/mzyLvKTywzsu2sRAqdjqsZCUCAwEAAaOBrjCBqzAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBTF +Xnzlk4tOSosBlUEM7AiDOYuW7TBbBgNVHREEVDBSgiJzdG9yYWdlLWdycGMueWRi +LnN2Yy5jbHVzdGVyLmxvY2FsgiwqLnN0b3JhZ2UtaW50ZXJjb25uZWN0LnlkYi5z +dmMuY2x1c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEAQYRvaGGFsnR4cp8Y +MJo948t7zI3Pgy20YonmYTriz1zeEYNj9+5t678p04FlCjIx0j4dad1tFC1bNtnI +FOJNMkhiyC1JPKSN7HR90L5P9JfsuunUVTNEHP6EuLj2/VnMXj+30qX9i0kVkcnH +OV9yXk8cpN0GuUUyfKUq5k/WS+/15JRoLJ9F8vS1lm77nHeQ9F7m1Yjqkc131N3k +czFLe3wexWhnKCnNw5OtdFgJsMgL09pYgnN+0xyE4iLY8f0o/b58J/tpbKkeJb9L +YogP1+ultfGvrlX8TE6iAEi8UA+rZCNpGwiqTI2Y9clZNnkYu7UgW7kn2ih5/vgU +xntdsA== +-----END CERTIFICATE----- diff --git a/e2e/tests/data/tls.key b/e2e/tests/data/tls.key new file mode 100644 index 00000000..c8999b40 --- /dev/null +++ b/e2e/tests/data/tls.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAyoZzKqbdQgRkIvcZ7xCHrc5TRDKQjfKa2P2AMnnu+1l1Q6kc +XXxCxfhSt/q8Y28jyBZcS09srwNyGqj7dF656OOrxpEWzVFo9VmhY8PyLvzanovr ++hfiTnMDC8J2MLbL9i5R/gZNmql/ANuFJ8f4R1pyt/CZiMvFhp6ESfvP0zrtHnBJ +6SvVsy7EEplbWxyWaKnclF/e4G4d8wuShilNrspwTJyf1iy1nWHZe5YUuTjYZLhs +xmwKrYTewSoPws7IIez+pZYYzdPHPp50tOLlTpe3M95bDHIzlbt86kLRDeLDnybI +ayYMOZSeDKd5L+bPIu8pPLDOy7axECp2OqxkJQIDAQABAoIBAQCXKyBPl9nTax+r +kbID5dzAeR9h6jRIH+xBR4cnJiih6LZE2LfZd+UHjEGCHl/8AHs+4KHnfNNtFy9W +gweeZw5xrW8MekQA4WFssYhrxVjChe5RJbPwK1+6mtKNNout9OPtT8nXyLCoXxfz +defAN900tWinr6mKmD9KKowoBROtYBsKVwO4DOl+JonZP7A5eF/ao4T8XFSOOju5 +aU9flCjw6iagzKuryrJCzt4LsTA3/svuMswPQ7LRMSJm+RsceiBl4oVCuRmJ79tN +pmhHZqG80upp/idgyuGZu+rgePiXSnmtn47iXXvCjZKTNMkabQMc2WZ5P3HWfG0/ +28kXH5IBAoGBAPswl0SrvZdcQbalNHjLc1JtMMhZs6LPZe+/QTP4i/5qXvee8Dox +vvGJlvjNQuOGNsPUTsLMY203eKQWJ4+soK2qDhuDCxGmyQJ7NgTTzdhOCULctXlD +pdBHxiSmCUGEtW7Goe3M1/CL3sVfRp85FzMYItHHZEMTHZNzoZ+XXAzBAoGBAM5n +SknfqHM4oBT0LxxkmVFyiFoCi2grXzsGjzK5rk30DWajwT5sAy9x+X2vmG2zKkBJ +tTBYYUFh+SlNtBsY/TW2ZIcmHDpu0w82zXXDLLhUwYjxudXzyG30yU8d4woAwE6u +NPiVmFX6C8U597a/nC36LLsrSnUPvaJ4voFyRlxlAoGAHJa8MLmnO2nppMMKxNDL +EE+TJMpo0pfuTyoiXqrkLBGpO1+gkc8Fn3H8d9bMzR6CbyljyXH/wvd0SKCo4gZQ +x1M6hdEVWm30JM8nJ8d/fyXqkeySzvlvDtSMbbFkDkvvZms/FNSioyMYOLiOTiLu +TAdsNxoNhEDRte2MMKDGfkECgYAGH1o8xr2gbVWSSYv8M5+4osUYpmqsNF0myxME +Vi2tckfTe5gH2fxeM+tKpyLGXkIqlgUh4f1Aiz9w0jU9eIhKR5bDy4Wa1h68nMuL +araw4RK8lS8GAa04VcKC7kgFy+/oZZJ8rTNPmZMvzoBik1x2oK0jAC29OzJM13gP +LuyXYQKBgQCvkar/nthVUlScVa3uPzqKdpNlqmeXQRMVwcmRQJcJoHYWkaF2RRAx +AcjWTkP6/ThUgPhI76O0neQqO2N0P+FnCkaE4qU6Wg+dZcH0q7HVnt2TRuCzsECQ +kxDqdbF+qbXmYfbJ4dKkJdF0dNad4/d2hL/wbvyHXDEEGpuIlZAZrg== +-----END RSA PRIVATE KEY----- diff --git a/e2e/tests/smoke_test.go b/e2e/tests/smoke_test.go index 5ba231f9..81eafdb2 100644 --- a/e2e/tests/smoke_test.go +++ b/e2e/tests/smoke_test.go @@ -551,6 +551,57 @@ var _ = Describe("Operator smoke test", func() { } }) + It("using grpcs for storage connection", func() { + By("create secret...") + cert := testobjects.DefaultCertificate( + filepath.Join(".", "data", "tls.crt"), + filepath.Join(".", "data", "tls.key"), + filepath.Join(".", "data", "ca.crt"), + ) + Expect(k8sClient.Create(ctx, cert)).Should(Succeed()) + defer func() { + Expect(k8sClient.Delete(ctx, cert)).Should(Succeed()) + }() + + By("create storage...") + storageSample = testobjects.DefaultStorage(filepath.Join(".", "data", "storage-block-4-2-config-tls.yaml")) + storageSample.Spec.Service.GRPC.TLSConfiguration.Enabled = true + storageSample.Spec.Service.GRPC.TLSConfiguration.Certificate = corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName}, + Key: "tls.crt", + } + storageSample.Spec.Service.GRPC.TLSConfiguration.Key = corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName}, + Key: "tls.key", + } + storageSample.Spec.Service.GRPC.TLSConfiguration.CertificateAuthority = corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName}, + Key: "ca.crt", + } + + Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed()) + defer func() { + Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed()) + }() + By("create database...") + Expect(k8sClient.Create(ctx, databaseSample)).Should(Succeed()) + defer func() { + Expect(k8sClient.Delete(ctx, databaseSample)).Should(Succeed()) + }() + + By("waiting until Storage is ready...") + waitUntilStorageReady(ctx, storageSample.Name, testobjects.YdbNamespace) + + By("checking that all the storage pods are running and ready...") + checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes) + + By("waiting until database is ready...") + waitUntilDatabaseReady(ctx, databaseSample.Name, testobjects.YdbNamespace) + + By("checking that all the database pods are running and ready...") + checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes) + }) + AfterEach(func() { Expect(uninstallOperatorWithHelm(testobjects.YdbNamespace)).Should(BeTrue()) Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed()) diff --git a/e2e/tests/test-objects/objects.go b/e2e/tests/test-objects/objects.go index a99b62af..d2b1e75e 100644 --- a/e2e/tests/test-objects/objects.go +++ b/e2e/tests/test-objects/objects.go @@ -11,12 +11,13 @@ import ( ) const ( - YdbImage = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17" - YdbNamespace = "ydb" - StorageName = "storage" - DatabaseName = "database" - DefaultDomain = "Root" - ReadyStatus = "Ready" + YdbImage = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17" + YdbNamespace = "ydb" + StorageName = "storage" + DatabaseName = "database" + CertificateSecretName = "storage-crt" + DefaultDomain = "Root" + ReadyStatus = "Ready" ) func constructAntiAffinityFor(key, value string) *corev1.Affinity { @@ -162,3 +163,25 @@ func DefaultDatabase() *v1alpha1.Database { }, } } + +func DefaultCertificate(certPath, keyPath, caPath string) *corev1.Secret { + cert, err := os.ReadFile(certPath) + Expect(err).To(BeNil()) + key, err := os.ReadFile(keyPath) + Expect(err).To(BeNil()) + ca, err := os.ReadFile(caPath) + Expect(err).To(BeNil()) + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: CertificateSecretName, + Namespace: YdbNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "ca.crt": ca, + "tls.crt": cert, + "tls.key": key, + }, + } +} diff --git a/internal/cms/tenant.go b/internal/cms/tenant.go index 265d364e..faad8c2f 100644 --- a/internal/cms/tenant.go +++ b/internal/cms/tenant.go @@ -31,6 +31,7 @@ func (t *Tenant) Create( ctx context.Context, database *resources.DatabaseBuilder, creds ydbCredentials.Credentials, + opts ...ydb.Option, ) error { logger := log.FromContext(ctx) createDatabaseURL := fmt.Sprintf( @@ -42,6 +43,7 @@ func (t *Tenant) Create( db, err := connection.Open(ctx, createDatabaseURL, ydb.WithCredentials(creds), + ydb.MergeOptions(opts...), ) if err != nil { logger.Error(err, "Error connecting to YDB storage") diff --git a/internal/connection/connection.go b/internal/connection/connection.go index f228835a..38e42767 100644 --- a/internal/connection/connection.go +++ b/internal/connection/connection.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "time" @@ -41,14 +42,26 @@ func Close(ctx context.Context, db *ydb.Driver) { } } -func LoadTLSCredentials(secure bool) grpc.DialOption { - if secure { - certPool, _ := x509.SystemCertPool() - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: certPool, +func LoadTLSCredentials(secure bool, caBundle []byte) (grpc.DialOption, error) { + if !secure { + return grpc.WithTransportCredentials(insecure.NewCredentials()), nil + } + var certPool *x509.CertPool + if len(caBundle) > 0 { + certPool = x509.NewCertPool() + if ok := certPool.AppendCertsFromPEM(caBundle); !ok { + return nil, errors.New("failed to parse CA bundle") + } + } else { + var err error + certPool, err = x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("failed to get system cert pool, error: %w", err) } - return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)) } - return grpc.WithTransportCredentials(insecure.NewCredentials()) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: certPool, + } + return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), nil } diff --git a/internal/controllers/database/init.go b/internal/controllers/database/init.go index 012c8797..3de92d90 100644 --- a/internal/controllers/database/init.go +++ b/internal/controllers/database/init.go @@ -178,8 +178,18 @@ func (r *Reconciler) handleTenantCreation( ) return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } + tlsOptions, err := resources.GetYDBTLSOption(ctx, database.Storage, r.Config) + if err != nil { + r.Recorder.Event( + database, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get YDB TLS options: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } - err = tenant.Create(ctx, database, creds) + err = tenant.Create(ctx, database, creds, tlsOptions) if err != nil { r.Recorder.Event( database, diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go index 9ed50500..2147a4fa 100644 --- a/internal/controllers/storage/sync.go +++ b/internal/controllers/storage/sync.go @@ -429,8 +429,18 @@ func (r *Reconciler) runSelfCheck( ) return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } + tlsOptions, err := resources.GetYDBTLSOption(ctx, storage.Unwrap(), r.Config) + if err != nil { + r.Recorder.Event( + storage, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get YDB TLS options: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } - result, err := healthcheck.GetSelfCheckResult(ctx, storage, creds) + result, err := healthcheck.GetSelfCheckResult(ctx, storage, creds, tlsOptions) if err != nil { r.Log.Error(err, "GetSelfCheckResult error") return Stop, ctrl.Result{RequeueAfter: SelfCheckRequeueDelay}, err diff --git a/internal/healthcheck/healthcheck.go b/internal/healthcheck/healthcheck.go index 7cd2e602..39de6dbe 100644 --- a/internal/healthcheck/healthcheck.go +++ b/internal/healthcheck/healthcheck.go @@ -15,7 +15,12 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) -func GetSelfCheckResult(ctx context.Context, cluster *resources.StorageClusterBuilder, creds ydbCredentials.Credentials) (*Ydb_Monitoring.SelfCheckResult, error) { +func GetSelfCheckResult( + ctx context.Context, + cluster *resources.StorageClusterBuilder, + creds ydbCredentials.Credentials, + opts ...ydb.Option, +) (*Ydb_Monitoring.SelfCheckResult, error) { logger := log.FromContext(ctx) getSelfCheckURL := fmt.Sprintf( "%s/%s", @@ -26,6 +31,7 @@ func GetSelfCheckResult(ctx context.Context, cluster *resources.StorageClusterBu db, err := connection.Open(ctx, getSelfCheckURL, ydb.WithCredentials(creds), + ydb.MergeOptions(opts...), ) if err != nil { return nil, err diff --git a/internal/resources/resource.go b/internal/resources/resource.go index c55f149d..a7e856cf 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/banzaicloud/k8s-objectmatcher/patch" + "github.com/ydb-platform/ydb-go-sdk/v3" ydbCredentials "github.com/ydb-platform/ydb-go-sdk/v3/credentials" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -341,6 +342,47 @@ func LabelExistsPredicate(selector labels.Selector) predicate.Predicate { }) } +func getYDBStaticCredentials( + ctx context.Context, + storage *api.Storage, + restConfig *rest.Config, +) (ydbCredentials.Credentials, error) { + auth := storage.Spec.OperatorConnection + username := auth.StaticCredentials.Username + password := api.DefaultRootPassword + if auth.StaticCredentials.Password != nil { + var err error + password, err = GetSecretKey( + ctx, + storage.Namespace, + restConfig, + auth.StaticCredentials.Password.SecretKeyRef, + ) + if err != nil { + return nil, fmt.Errorf( + "failed to get password for StaticCredentials from secret: %s, key: %s, error: %w", + auth.StaticCredentials.Password.SecretKeyRef.Name, + auth.StaticCredentials.Password.SecretKeyRef.Key, + err) + } + } + endpoint := storage.GetStorageEndpoint() + + var caBundle []byte + if storage.IsStorageEndpointSecure() { + var err error + caBundle, err = getStorageGrpcServiceCABundle(ctx, storage, restConfig) + if err != nil { + return nil, err + } + } + dialOptions, err := connection.LoadTLSCredentials(storage.IsStorageEndpointSecure(), caBundle) + if err != nil { + return nil, err + } + return ydbCredentials.NewStaticCredentials(username, password, endpoint, dialOptions), nil +} + func GetYDBCredentials( ctx context.Context, storage *api.Storage, @@ -370,32 +412,53 @@ func GetYDBCredentials( } if auth.StaticCredentials != nil { - username := auth.StaticCredentials.Username - password := api.DefaultRootPassword - if auth.StaticCredentials.Password != nil { - var err error - password, err = GetSecretKey( - ctx, - storage.Namespace, - restConfig, - auth.StaticCredentials.Password.SecretKeyRef, - ) - if err != nil { - return nil, fmt.Errorf( - "failed to get password for StaticCredentials from secret: %s, key: %s, error: %w", - auth.StaticCredentials.Password.SecretKeyRef.Name, - auth.StaticCredentials.Password.SecretKeyRef.Key, - err) - } - } - endpoint := storage.GetStorageEndpoint() - secure := connection.LoadTLSCredentials(storage.IsStorageEndpointSecure()) - return ydbCredentials.NewStaticCredentials(username, password, endpoint, secure), nil + return getYDBStaticCredentials(ctx, storage, restConfig) } return nil, errors.New("unsupported auth type for GetYDBCredentials") } +func getStorageGrpcServiceCABundle( + ctx context.Context, + storage *api.Storage, + restConfig *rest.Config, +) ([]byte, error) { + if !storage.IsStorageEndpointSecure() { + return nil, errors.New("can't get storage grpc CA for insecure endpoint") + } + + tlsConfig := storage.Spec.Service.GRPC.TLSConfiguration + caBody, err := GetSecretKey( + ctx, + storage.Namespace, + restConfig, + &tlsConfig.CertificateAuthority, + ) + if err != nil { + return nil, fmt.Errorf( + "failed to get CA for storage grpc service from secret: %s, key: %s, error: %w", + tlsConfig.CertificateAuthority.Name, + tlsConfig.CertificateAuthority.Key, + err) + } + return []byte(caBody), nil +} + +func GetYDBTLSOption( + ctx context.Context, + storage *api.Storage, + restConfig *rest.Config, +) (ydb.Option, error) { + if !storage.IsStorageEndpointSecure() { + return ydb.WithInsecure(), nil + } + caBundle, err := getStorageGrpcServiceCABundle(ctx, storage, restConfig) + if err != nil { + return nil, err + } + return ydb.WithCertificatesFromPem(caBundle), nil +} + func buildCAStorePatchingCommandArgs( caBundle string, grpcService api.GRPCService,