Add ceph client configuration

openstack_hypervisor/hooks.py

@@ -91,6 +91,16 @@
     Path("lib/neutron"),
     Path("run/hypervisor-config"),
 ]
+SECRET_XML = string.Template(
+    """
+<secret ephemeral='no' private='no'>
+   <uuid>$uuid</uuid>
+   <usage type='ceph'>
+     <name>client.cinder-ceph secret</name>
+   </usage>
+</secret>
+"""
+)
 
 
 def _generate_secret(length: int = DEFAULT_SECRET_LENGTH) -> str:
@@ -223,6 +233,9 @@ def _get_local_ip_by_default_route() -> str:
     "compute.virt-type": "auto",
     "compute.cpu-models": UNSET,
     "compute.spice-proxy-address": _get_local_ip_by_default_route,  # noqa: F821
+    "compute.rbd_user": "nova",
+    "compute.rbd_secret_uuid": UNSET,
+    "compute.rbd_key": UNSET,
     # Neutron
     "network.physnet-name": "physnet1",
     "network.external-bridge": "br-ex",
@@ -899,6 +912,72 @@ def _is_hw_virt_supported() -> bool:
         return False
 
 
+def _set_secret(conn, secret_uuid: str, secret_value: str) -> None:
+    """Set the ceph access secret in libvirt."""
+    logging.info(f"Setting secret {secret_uuid}")
+    new_secret = conn.secretDefineXML(SECRET_XML.substitute(uuid=secret_uuid))
+    # nova assumes the secret is raw and always encodes it *1, so decode it
+    # before storing it.
+    # *1 https://opendev.org/openstack/nova/src/branch/stable/2023.1/nova/
+    #           virt/libvirt/imagebackend.py#L1110
+    new_secret.setValue(base64.b64decode(secret_value))
+
+
+def _get_libvirt():
+    # Lazy import libvirt otherwise snap will not build
+    import libvirt
+
+    return libvirt
+
+
+def _ensure_secret(secret_uuid: str, secret_value: str) -> None:
+    """Ensure libvirt has the ceph access secret with the correct value."""
+    libvirt = _get_libvirt()
+    conn = libvirt.open("qemu:///system")
+    # Check if secret exists
+    if secret_uuid in conn.listSecrets():
+        logging.info(f"Found secret {secret_uuid}")
+        # check secret matches
+        secretobj = conn.secretLookupByUUIDString(secret_uuid)
+        try:
+            secret = secretobj.value()
+        except libvirt.libvirtError as e:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_SECRET:
+                logging.info(f"Secret {secret_uuid} has no value.")
+                secret = None
+            else:
+                raise
+        # Secret is stored raw so encode it before comparison.
+        if secret == base64.b64encode(secret_value.encode()):
+            logging.info(f"Secret {secret_uuid} has desired value.")
+        else:
+            logging.info(f"Secret {secret_uuid} has wrong value, replacing.")
+            secretobj.undefine()
+            _set_secret(conn, secret_uuid, secret_value)
+    else:
+        logging.info(f"Secret {secret_uuid} not found, creating.")
+        _set_secret(conn, secret_uuid, secret_value)
+
+
+def _configure_ceph(snap) -> None:
+    """Configure ceph client.
+
+    :param snap: the snap reference
+    :type snap: Snap
+    :return: None
+    """
+    logging.info("Configuring ceph access")
+    context = (
+        snap.config.get_options(
+            "compute",
+        )
+        .as_dict()
+        .get("compute")
+    )
+    if all(k in context for k in ("rbd-key", "rbd-secret-uuid")):
+        _ensure_secret(context["rbd-secret-uuid"], context["rbd-key"])
+
+
 def _configure_kvm(snap) -> None:
     """Configure KVM hardware virtualization.
 
@@ -1021,3 +1100,4 @@ def configure(snap: Snap) -> None:
     _configure_ovn_external_networking(snap)
     _configure_ovn_tls(snap)
     _configure_kvm(snap)
+    _configure_ceph(snap)

openstack_hypervisor/model.py

@@ -73,6 +73,9 @@ class ComputeConfig(BaseModel):
     virt_type: str = Field(alias="virt-type", default="auto")
     cpu_models: Optional[str] = Field(alias="cpu-models")
     spice_proxy_address: Optional[IPvAnyAddress] = Field(alias="spice-proxy-address")
+    rbd_user: Optional[str] = Field(alias="rbd-user", default="nova")
+    rbd_secret_uuid: Optional[str] = Field(alias="rbd-secret-uuid")
+    rbd_key: Optional[str] = Field(alias="rbd-key")
 
 
 class NetworkConfig(BaseModel):

templates/nova.conf.j2

@@ -33,6 +33,10 @@ cpu_mode = {{ compute.cpu_mode }}
 cpu_models = {{ compute.cpu_models }}
 {% endif %}
 
+{% if compute.rbd_secret_uuid %}
+rbd_user = {{ compute.rbd_user }}
+rbd_secret_uuid = {{ compute.rbd_secret_uuid }}
+{% endif %}
 
 [oslo_concurrency]
 # Oslo Concurrency lock path

tests/unit/test_hooks.py

@@ -317,3 +317,70 @@ def test_del_external_nics_from_bridge(self, mocker):
         hooks._del_external_nics_from_bridge("br-ex")
         expect = [mock.call("br-ex", "eth0"), mock.call("br-ex", "eth1")]
         mock_del_interface_from_bridge.assert_has_calls(expect)
+
+    def test_set_secret(self, mocker):
+        conn_mock = mocker.Mock()
+        secret_mock = mocker.Mock()
+        conn_mock.secretDefineXML.return_value = secret_mock
+        hooks._set_secret(conn_mock, "uuid1", "c2VjcmV0Cg==")
+        conn_mock.secretDefineXML.assert_called_once()
+        secret_mock.setValue.assert_called_once_with(b"secret\n")
+
+    def test_ensure_secret_new_secret(self, mocker):
+        conn_mock = mocker.Mock()
+        mock_libvirt = mocker.Mock()
+        mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt")
+        mock_get_libvirt.return_value = mock_libvirt
+        mock_libvirt.open.return_value = conn_mock
+        mock_set_secret = mocker.patch.object(hooks, "_set_secret")
+        conn_mock.listSecrets.return_value = []
+        hooks._ensure_secret("uuid1", "secret")
+        mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret")
+
+    def test_ensure_secret_secret_exists(self, mocker):
+        conn_mock = mocker.Mock()
+        mock_libvirt = mocker.Mock()
+        secret_mock = mocker.Mock()
+        secret_mock.value.return_value = b"c2VjcmV0"
+        mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt")
+        mock_get_libvirt.return_value = mock_libvirt
+        mock_libvirt.open.return_value = conn_mock
+        mock_set_secret = mocker.patch.object(hooks, "_set_secret")
+        conn_mock.listSecrets.return_value = ["uuid1"]
+        conn_mock.secretLookupByUUIDString.return_value = secret_mock
+        hooks._ensure_secret("uuid1", "secret")
+        assert not mock_set_secret.called
+
+    def test_ensure_secret_secret_wrong_value(self, mocker):
+        conn_mock = mocker.Mock()
+        mock_libvirt = mocker.Mock()
+        secret_mock = mocker.Mock()
+        secret_mock.value.return_value = b"wrong"
+        mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt")
+        mock_get_libvirt.return_value = mock_libvirt
+        mock_libvirt.open.return_value = conn_mock
+        mock_set_secret = mocker.patch.object(hooks, "_set_secret")
+        conn_mock.listSecrets.return_value = ["uuid1"]
+        conn_mock.secretLookupByUUIDString.return_value = secret_mock
+        hooks._ensure_secret("uuid1", "secret")
+        mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret")
+
+    def test_ensure_secret_secret_missing_value(self, mocker):
+        class FakeError(Exception):
+            def get_error_code(self):
+                return 42
+
+        conn_mock = mocker.Mock()
+        mock_libvirt = mocker.Mock()
+        mock_libvirt.libvirtError = FakeError
+        mock_libvirt.VIR_ERR_NO_SECRET = 42
+        secret_mock = mocker.Mock()
+        secret_mock.value.side_effect = FakeError()
+        mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt")
+        mock_get_libvirt.return_value = mock_libvirt
+        mock_libvirt.open.return_value = conn_mock
+        mock_set_secret = mocker.patch.object(hooks, "_set_secret")
+        conn_mock.listSecrets.return_value = ["uuid1"]
+        conn_mock.secretLookupByUUIDString.return_value = secret_mock
+        hooks._ensure_secret("uuid1", "secret")
+        mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret")