ceph-iscsi: add erasure pool support

Erasure coded pools do not support omap, will just store the data
in ec pool, and their metadata will be in 'rbd' replicated pool.

Signed-off-by: Xiubo Li <[email protected]>

README

@@ -95,6 +95,11 @@ curl --user admin:admin -d ip_address=2006:ac81::1104 \
 NOTE: please make sure both the IPv4 and IPv6 addresses are in the trusted
 ip list in iscsi-gateway.cfg.
 
+Erasure Pool Support:
+For the erasure pool, there has no any special thing need to do beforehand, you can just
+use it like the replicated pools. But internally the ceph-iscsi will use the replicated
+'rbd' pool to save the meta data as defualt and use the erasure pool to save the data for
+all the images.
 
 ## Installation
 ### Via RPM

ceph_iscsi_config/lun.py

@@ -1,3 +1,4 @@
+import json
 import rados
 import rbd
 import re
@@ -13,8 +14,9 @@
 from ceph_iscsi_config.backstore import USER_RBD
 from ceph_iscsi_config.utils import (convert_2_bytes, gen_control_string,
                                      valid_size, get_pool_id, ip_addresses,
-                                     get_pools, get_rbd_size, this_host,
-                                     human_size, CephiSCSIError)
+                                     get_pools, get_rbd_size, run_shell_cmd,
+                                     human_size, CephiSCSIError, this_host,
+                                     parse_disk_meta)
 from ceph_iscsi_config.gateway_object import GWObject
 from ceph_iscsi_config.target import GWTarget
 from ceph_iscsi_config.client import GWClient, CHAP
@@ -46,13 +48,14 @@ class RBDDev(object):
         ]
     }
 
-    def __init__(self, image, size, backstore, pool=None):
+    def __init__(self, image, size, backstore, pool=None, ecpool=None):
         self.image = image
         self.size_bytes = convert_2_bytes(size)
         self.backstore = backstore
         if pool is None:
             pool = settings.config.pool
         self.pool = pool
+        self.ecpool = ecpool
         self.pool_id = get_pool_id(pool_name=self.pool)
         self.error = False
         self.error_msg = ''
@@ -74,13 +77,14 @@ def create(self):
                                     self.image,
                                     self.size_bytes,
                                     features=RBDDev.default_features(self.backstore),
-                                    old_format=False)
+                                    old_format=False,
+                                    data_pool=self.ecpool)
 
                 except (rbd.ImageExists, rbd.InvalidArgument) as err:
                     self.error = True
                     self.error_msg = ("Failed to create rbd image {} in "
-                                      "pool {} : {}".format(self.image,
-                                                            self.pool,
+                                      "pool {}, ecpool {} : {}".format(self.image,
+                                                            self.pool, self.ecpool,
                                                             err))
 
     def delete(self):
@@ -289,14 +293,18 @@ class LUN(GWObject):
         USER_RBD: TCMU_SETTINGS
     }
 
-    def __init__(self, logger, pool, image, size, allocating_host,
+    def __init__(self, logger, pool, ecpool, image, size, allocating_host,
                  backstore, backstore_object_name):
         self.logger = logger
         self.image = image
         self.pool = pool
+        self.ecpool = ecpool
         self.pool_id = 0
         self.size_bytes = convert_2_bytes(size)
-        self.config_key = '{}/{}'.format(self.pool, self.image)
+        if ecpool:
+            self.config_key = '{}/{}/{}'.format(pool, ecpool, image)
+        else:
+            self.config_key = '{}/{}'.format(pool, image)
 
         self.allocating_host = allocating_host
         self.backstore = backstore
@@ -351,7 +359,7 @@ def remove_lun(self, preserve_image):
             if self.error:
                 return
 
-        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool, self.ecpool)
 
         if local_gw == self.allocating_host:
             # by using the allocating host we ensure the delete is not
@@ -574,6 +582,38 @@ def activate(self):
             if client_err:
                 raise CephiSCSIError(client_err)
 
+    def _erasure_pool_check(self):
+        if not self.ecpool:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd metadata --format=json".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf))
+        if err:
+            self.logger.error("Cannot get the objectstore type")
+            return err
+        bluestore = False
+        for _osd in json.loads(data):
+            store_type = _osd['osd_objectstore']
+            self.logger.debug(f"pool ({self.pool}) objectstore type is ({store_type})")
+            if store_type is 'bluestore':
+                bluestore = True
+                break
+
+        if not bluestore:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd pool get {pool} allow_ec_overwrites".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf, pool=self.ecpool))
+        if err:
+            self.logger.error(f"Cannot get allow_ec_overwrites from pool ({self.pool})")
+            return err
+        self.logger.debug(f"erasure pool ({self.pool}) allow_ec_overwrites is enabled")
+        return None
+
     def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         """
         Create image and add to LIO and config.
@@ -583,6 +623,10 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         :return: LIO storage object if successful and keep_dev_in_lio=True
                  else None.
         """
+        err = self._erasure_pool_check()
+        if err:
+            return None
+
         self.logger.debug("LUN.allocate starting, listing rbd devices")
         disk_list = RBDDev.rbd_list(pool=self.pool)
         self.logger.debug("rados pool '{}' contains the following - "
@@ -593,7 +637,8 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                           "allocations is {}".format(local_gw,
                                                      self.allocating_host))
 
-        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool,
+                           self.ecpool)
         self.pool_id = rbd_image.pool_id
 
         # if the image required isn't defined, create it!
@@ -703,6 +748,7 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                     disk_attr = {"wwn": wwn,
                                  "image": self.image,
                                  "pool": self.pool,
+                                 "ecpool": self.ecpool,
                                  "allocating_host": self.allocating_host,
                                  "pool_id": rbd_image.pool_id,
                                  "controls": self.controls,
@@ -1112,7 +1158,7 @@ def _backstore_object_name_exists(disks_config, backstore_object_name_exists):
                     if disk['backstore_object_name'] == backstore_object_name_exists]) > 0
 
     @staticmethod
-    def get_backstore_object_name(pool, image, disks_config):
+    def get_backstore_object_name(pool, ecpool, image, disks_config):
         """
         Determine the backstore storage object name based on the pool name,
         image name, and existing storage object names to avoid conflicts.
@@ -1126,7 +1172,10 @@ def get_backstore_object_name(pool, image, disks_config):
         :param disks_config: disks configuration from `gateway.conf`
         :return: the backstore storage object name to be used
         """
-        base_name = '{}.{}'.format(pool, image)
+        if ecpool:
+            base_name = '{}.{}.{}'.format(pool, ecpool, image)
+        else:
+            base_name = '{}.{}'.format(pool, image)
         candidate = base_name
         counter = 0
         while LUN._backstore_object_name_exists(disks_config, candidate):
@@ -1230,14 +1279,15 @@ def define_luns(logger, config, target):
                                   if disk_key.startswith(pool + '/')]
                     for disk_key in pool_disks:
 
-                        pool, image_name = disk_key.split('/')
+                        pool, ecpool, image_name = parse_disk_meta(disk_key)
+
                         with rbd.Image(ioctx, image_name) as rbd_image:
 
                             disk_config = config.config['disks'][disk_key]
                             backstore = disk_config['backstore']
                             backstore_object_name = disk_config['backstore_object_name']
 
-                            lun = LUN(logger, pool, image_name,
+                            lun = LUN(logger, pool, ecpool, image_name,
                                       rbd_image.size(), local_gw, backstore,
                                       backstore_object_name)
 

ceph_iscsi_config/utils.py

@@ -6,6 +6,7 @@
 import re
 import datetime
 import os
+import json
 
 import ceph_iscsi_config.settings as settings
 
@@ -27,6 +28,16 @@ class CephiSCSIInval(CephiSCSIError):
     '''
     pass
 
+def parse_disk_meta(disk):
+    pool = None
+    ecpool = None
+    image = None
+    try:
+        pool, ecpool, image = disk.split('/')
+    except ValueError:
+        pool, image = disk.split('/')
+        pass
+    return pool, ecpool, image
 
 def run_shell_cmd(cmd, stderr=None, shell=True):
     if not stderr:

gwcli/client.py

@@ -540,7 +540,7 @@ def ui_command_disk(self, action='add', disk=None, size=None):
 
                 # a disk given here would be of the form pool.image
                 try:
-                    pool, image = disk.split('/')
+                    pool, ecpool, image = parse_disk_meta(disk)
                 except ValueError:
                     self.logger.error("Invalid format. Use pool_name/disk_name")
                     return