Add ceph client configuration

openstack_hypervisor/api.py

@@ -31,6 +31,8 @@
     "network": model.NetworkConfig,
     "node": model.NodeConfig,
     "logging": model.LoggingConfig,
+    "telemetry": model.TelemetryConfig,
+    "monitoring": model.MonitoringConfig,
 }
 
 
@@ -138,6 +140,18 @@ async def update_logging(config: model.LoggingConfig):
     return _update_settings("logging", config)
 
 
+@app.patch("/settings/telemetry")
+async def update_telemetry(config: model.TelemetryConfig):
+    """Updates telemetry section settings."""
+    return _update_settings("telemetry", config)
+
+
+@app.patch("/settings/monitoring")
+async def update_monitoring(config: model.MonitoringConfig):
+    """Updates monitoring section settings."""
+    return _update_settings("monitoring", config)
+
+
 @app.post("/reset")
 async def reset_config():
     """Reset all configs to default."""

openstack_hypervisor/hooks.py

@@ -70,6 +70,7 @@
     Path("etc/neutron/neutron.conf.d"),
     Path("etc/ssl/certs"),
     Path("etc/ssl/private"),
+    Path("etc/ceilometer"),
     # log
     Path("log/libvirt/qemu"),
     Path("log/ovn"),
@@ -91,6 +92,22 @@
     Path("lib/neutron"),
     Path("run/hypervisor-config"),
 ]
+SECRET_XML = string.Template(
+    """
+<secret ephemeral='no' private='no'>
+   <uuid>$uuid</uuid>
+   <usage type='ceph'>
+     <name>client.cinder-ceph secret</name>
+   </usage>
+</secret>
+"""
+)
+
+# As defined in the snap/snapcraft.yaml
+MONITORING_SERVICES = [
+    "libvirt-exporter",
+    "ovs-exporter",
+]
 
 
 def _generate_secret(length: int = DEFAULT_SECRET_LENGTH) -> str:
@@ -223,6 +240,9 @@ def _get_local_ip_by_default_route() -> str:
     "compute.virt-type": "auto",
     "compute.cpu-models": UNSET,
     "compute.spice-proxy-address": _get_local_ip_by_default_route,  # noqa: F821
+    "compute.rbd_user": "nova",
+    "compute.rbd_secret_uuid": UNSET,
+    "compute.rbd_key": UNSET,
     # Neutron
     "network.physnet-name": "physnet1",
     "network.external-bridge": "br-ex",
@@ -236,11 +256,16 @@ def _get_local_ip_by_default_route() -> str:
     "network.enable-gateway": False,
     "network.ip-address": _get_local_ip_by_default_route,  # noqa: F821
     "network.external-nic": UNSET,
+    # Monitoring
+    "monitoring.enable": False,
     # General
     "logging.debug": False,
     "node.fqdn": socket.getfqdn,
     "node.ip-address": _get_local_ip_by_default_route,  # noqa: F821
     # TLS
+    # Telemetry
+    "telemetry.enable": False,
+    "telemetry.publisher-secret": UNSET,
 }
 
 
@@ -256,6 +281,12 @@ def _get_local_ip_by_default_route() -> str:
         "network",
     ],
     "neutron-ovn-metadata-agent": ["credentials", "network", "node", "network.ovn_key"],
+    "ceilometer-compute-agent": [
+        "identity.password",
+        "identity.username",
+        "identity",
+        "rabbitmq.url",
+    ],
 }
 
 
@@ -335,6 +366,14 @@ def _context_compat(context: Dict[str, Any]) -> Dict[str, Any]:
     Path("etc/openvswitch/system-id.conf"): {
         "template": "system-id.conf.j2",
     },
+    Path("etc/ceilometer/ceilometer.conf"): {
+        "template": "ceilometer.conf.j2",
+        "services": ["ceilometer-compute-agent"],
+    },
+    Path("etc/ceilometer/polling.yaml"): {
+        "template": "polling.yaml.j2",
+        "services": ["ceilometer-compute-agent"],
+    },
 }
 
 
@@ -899,7 +938,73 @@ def _is_hw_virt_supported() -> bool:
         return False
 
 
-def _configure_kvm(snap) -> None:
+def _set_secret(conn, secret_uuid: str, secret_value: str) -> None:
+    """Set the ceph access secret in libvirt."""
+    logging.info(f"Setting secret {secret_uuid}")
+    new_secret = conn.secretDefineXML(SECRET_XML.substitute(uuid=secret_uuid))
+    # nova assumes the secret is raw and always encodes it *1, so decode it
+    # before storing it.
+    # *1 https://opendev.org/openstack/nova/src/branch/stable/2023.1/nova/
+    #           virt/libvirt/imagebackend.py#L1110
+    new_secret.setValue(base64.b64decode(secret_value))
+
+
+def _get_libvirt():
+    # Lazy import libvirt otherwise snap will not build
+    import libvirt
+
+    return libvirt
+
+
+def _ensure_secret(secret_uuid: str, secret_value: str) -> None:
+    """Ensure libvirt has the ceph access secret with the correct value."""
+    libvirt = _get_libvirt()
+    conn = libvirt.open("qemu:///system")
+    # Check if secret exists
+    if secret_uuid in conn.listSecrets():
+        logging.info(f"Found secret {secret_uuid}")
+        # check secret matches
+        secretobj = conn.secretLookupByUUIDString(secret_uuid)
+        try:
+            secret = secretobj.value()
+        except libvirt.libvirtError as e:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_SECRET:
+                logging.info(f"Secret {secret_uuid} has no value.")
+                secret = None
+            else:
+                raise
+        # Secret is stored raw so encode it before comparison.
+        if secret == base64.b64encode(secret_value.encode()):
+            logging.info(f"Secret {secret_uuid} has desired value.")
+        else:
+            logging.info(f"Secret {secret_uuid} has wrong value, replacing.")
+            secretobj.undefine()
+            _set_secret(conn, secret_uuid, secret_value)
+    else:
+        logging.info(f"Secret {secret_uuid} not found, creating.")
+        _set_secret(conn, secret_uuid, secret_value)
+
+
+def _configure_ceph(snap) -> None:
+    """Configure ceph client.
+
+    :param snap: the snap reference
+    :type snap: Snap
+    :return: None
+    """
+    logging.info("Configuring ceph access")
+    context = (
+        snap.config.get_options(
+            "compute",
+        )
+        .as_dict()
+        .get("compute")
+    )
+    if all(k in context for k in ("rbd-key", "rbd-secret-uuid")):
+        _ensure_secret(context["rbd-secret-uuid"], context["rbd-key"])
+
+
+def _configure_kvm(snap: Snap) -> None:
     """Configure KVM hardware virtualization.
 
     :param snap: the snap reference
@@ -919,6 +1024,25 @@ def _configure_kvm(snap) -> None:
         snap.config.set({"compute.virt-type": "qemu"})
 
 
+def _configure_monitoring_services(snap: Snap) -> None:
+    """Configure all the monitoring services.
+
+    :param snap: the snap reference
+    :type snap: Snap
+    :return: None
+    """
+    services = snap.services.list()
+    enable_monitoring = snap.config.get("monitoring.enable")
+    if enable_monitoring:
+        logging.info("Enabling all exporter services.")
+        for service in MONITORING_SERVICES:
+            services[service].start(enable=True)
+    else:
+        logging.info("Disabling all exporter services.")
+        for service in MONITORING_SERVICES:
+            services[service].stop(disable=True)
+
+
 def services() -> List[str]:
     """List of services managed by hooks."""
     return sorted(list(set([w for v in TEMPLATES.values() for w in v.get("services", [])])))
@@ -958,6 +1082,15 @@ def _services_not_ready(context: dict) -> List[str]:
     return sorted(list(set(not_ready)))
 
 
+def _services_not_enabled_by_config(context: dict) -> List[str]:
+    """Check if services are enabled by configuration."""
+    not_enabled = []
+    if not context.get("telemetry", {}).get("enable"):
+        not_enabled.append("ceilometer-compute-agent")
+
+    return not_enabled
+
+
 def configure(snap: Snap) -> None:
     """Runs the `configure` hook for the snap.
 
@@ -984,6 +1117,8 @@ def configure(snap: Snap) -> None:
         "node",
         "rabbitmq",
         "credentials",
+        "telemetry",
+        "monitoring",
     ).as_dict()
 
     # Add some general snap path information
@@ -997,6 +1132,7 @@ def configure(snap: Snap) -> None:
     context = _context_compat(context)
     logging.info(context)
     exclude_services = _services_not_ready(context)
+    exclude_services.extend(_services_not_enabled_by_config(context))
     logging.warning(f"{exclude_services} are missing required config, stopping")
     services = snap.services.list()
     for service in exclude_services:
@@ -1021,3 +1157,5 @@ def configure(snap: Snap) -> None:
     _configure_ovn_external_networking(snap)
     _configure_ovn_tls(snap)
     _configure_kvm(snap)
+    _configure_monitoring_services(snap)
+    _configure_ceph(snap)

openstack_hypervisor/model.py

@@ -73,6 +73,9 @@ class ComputeConfig(BaseModel):
     virt_type: str = Field(alias="virt-type", default="auto")
     cpu_models: Optional[str] = Field(alias="cpu-models")
     spice_proxy_address: Optional[IPvAnyAddress] = Field(alias="spice-proxy-address")
+    rbd_user: Optional[str] = Field(alias="rbd-user", default="nova")
+    rbd_secret_uuid: Optional[str] = Field(alias="rbd-secret-uuid")
+    rbd_key: Optional[str] = Field(alias="rbd-key")
 
 
 class NetworkConfig(BaseModel):
@@ -103,3 +106,16 @@ class LoggingConfig(BaseModel):
     """Data model for the logging configuration for the hypervisor."""
 
     debug: bool = Field(default=False)
+
+
+class TelemetryConfig(BaseModel):
+    """Data model for telemetry configuration settings."""
+
+    enable: bool = Field(default=False)
+    publisher_secret: Optional[str] = Field(alias="publisher-secret")
+
+
+class MonitoringConfig(BaseModel):
+    """Data model for the monitoring configuration settings."""
+
+    enable: bool = Field(default=False)

openstack_hypervisor/services.py

@@ -35,6 +35,7 @@ class OpenStackService:
 
     conf_files = []
     conf_dirs = []
+    extra_args = []
 
     executable = None
 
@@ -70,6 +71,7 @@ def run(self, snap: Snap) -> int:
 
         cmd = [str(executable)]
         cmd.extend(args)
+        cmd.extend(self.extra_args)
         completed_process = subprocess.run(cmd)
 
         logging.info(f"Exiting with code {completed_process.returncode}")
@@ -125,6 +127,21 @@ class NeutronOVNMetadataAgentService(OpenStackService):
 neutron_ovn_metadata_agent = partial(entry_point, NeutronOVNMetadataAgentService)
 
 
+class CeilometerComputeAgentService(OpenStackService):
+    """A python service object used to run the ceilometer-agent-compute daemon."""
+
+    conf_files = [
+        Path("etc/ceilometer/ceilometer.conf"),
+    ]
+    conf_dirs = []
+    extra_args = ["--polling-namespaces", "compute"]
+
+    executable = Path("usr/bin/ceilometer-polling")
+
+
+ceilometer_compute_agent = partial(entry_point, CeilometerComputeAgentService)
+
+
 class OVSDBServerService:
     """A python service object used to run the ovsdb-server daemon."""
 
@@ -157,3 +174,56 @@ def run(self, snap: Snap) -> int:
 
 
 ovsdb_server = partial(entry_point, OVSDBServerService)
+
+
+class OVSExporterService:
+    """A python service object used to run the ovs-exporter daemon."""
+
+    def run(self, snap: Snap) -> int:
+        """Runs the ovs-exporter service.
+
+        Invoked when config monitoring is enable.
+
+        :param snap: the snap context
+        :type snap: Snap
+        :return: exit code of the process
+        :rtype: int
+        """
+        setup_logging(snap.paths.common / "ovs-exporter.log")
+        executable = snap.paths.snap / "bin" / "ovs-exporter"
+        listen_address = ":9475"
+        args = [
+            f"-web.listen-address={listen_address}",
+            "-database.vswitch.file.data.path",
+            f"{snap.paths.common}/etc/openvswitch/conf.db",
+            "-database.vswitch.file.log.path",
+            f"{snap.paths.common}/log/openvswitch/ovsdb-server.log",
+            "-database.vswitch.file.pid.path",
+            f"{snap.paths.common}/run/openvswitch/ovsdb-server.pid",
+            "-database.vswitch.file.system.id.path",
+            f"{snap.paths.common}/etc/openvswitch/system-id.conf",
+            "-database.vswitch.name",
+            "Open_vSwitch",
+            "-database.vswitch.socket.remote",
+            "unix:" + f"{snap.paths.common}/run/openvswitch/db.sock",
+            "-service.ovncontroller.file.log.path",
+            f"{snap.paths.common}/log/ovn/ovn-controller.log",
+            "-service.ovncontroller.file.pid.path",
+            f"{snap.paths.common}/run/ovn/ovn-controller.pid",
+            "-service.vswitchd.file.log.path",
+            f"{snap.paths.common}/log/openvswitch/ovs-vswitchd.log",
+            "-service.vswitchd.file.pid.path",
+            f"{snap.paths.common}/run/openvswitch/ovs-vswitchd.pid",
+            "-system.run.dir",
+            f"{snap.paths.common}/run/openvswitch",
+        ]
+        cmd = [str(executable)]
+        cmd.extend(args)
+
+        completed_process = subprocess.run(cmd)
+
+        logging.info(f"Exiting with code {completed_process.returncode}")
+        return completed_process.returncode
+
+
+ovs_exporter = partial(entry_point, OVSExporterService)

setup.cfg

@@ -30,6 +30,8 @@ console_scripts =
     nova-api-metadata-service = openstack_hypervisor.services:nova_api_metadata
     ovsdb-server-service = openstack_hypervisor.services:ovsdb_server
     neutron-ovn-metadata-agent-service = openstack_hypervisor.services:neutron_ovn_metadata_agent
+    ceilometer-compute-agent-service = openstack_hypervisor.services:ceilometer_compute_agent
+    ovs-exporter-service = openstack_hypervisor.services:ovs_exporter
 
 snaphelpers.hooks =
     configure = openstack_hypervisor.hooks:configure