ceph-iscsi: add erasure pool support

The erasure coded pool does not support omap, here we will just store the data in it and will store the metadata in a replicated pool. This will add "ecpool=<ec_name>" parameter to specify the erasure pool to store the data when creating a disk, and the "pool=<name>", which is "rbd" as default, will be used to store the metadata. For rest API when creating the disk and disksnap, you must replace the "disk" to "ecdisk" in the URL, more detail please see the README. Signed-off-by: Xiubo Li <[email protected]>
ceph · Nov 22, 2021 · f1b8906 · f1b8906
1 parent 7ec4057
commit f1b8906
Show file tree

Hide file tree

Showing 10 changed files with 404 additions and 159 deletions.
diff --git a/README b/README
@@ -17,17 +17,17 @@ o- / ...........................................................................
   o- cluster .................................................................. [Clusters: 1]
   | o- ceph ..................................................................... [HEALTH_OK]
   |   o- pools ................................................................... [Pools: 3]
-  |   | o- ec ........................................ [(2+1), Commit: 0b/40G (0%), Used: 0b]
+  |   | o- ec ....................................... [(2+2), Commit: 0b/40G (0%), Used: 24K]
   |   | o- iscsi ..................................... [(x3), Commit: 0b/20G (0%), Used: 18b]
   |   | o- rbd ....................................... [(x3), Commit: 8G/20G (40%), Used: 5K]
   |   o- topology ......................................................... [OSDs: 3,MONs: 3]
   o- disks ................................................................... [8G, Disks: 5]
   | o- rbd ....................................................................... [rbd (8G)]
-  |   o- disk_1 ............................................................... [disk_1 (1G)]
-  |   o- disk_2 ............................................................... [disk_2 (2G)]
-  |   o- disk_3 ............................................................... [disk_3 (2G)]
-  |   o- disk_4 ............................................................... [disk_4 (1G)]
-  |   o- disk_5 ............................................................... [disk_5 (2G)]
+  |   o- disk_1 ........................................................... [rbd/disk_1 (1G)]
+  |   o- disk_2 ........................................................... [rbd/disk_2 (2G)]
+  |   o- disk_3 ........................................................... [rbd/disk_3 (2G)]
+  |   o- disk_4 ........................................................ [rbd/ec/disk_4 (1G)]
+  |   o- disk_5 ........................................................ [rbd/ec/disk_5 (2G)]
   o- iscsi-targets ............................................................. [Targets: 1]
     o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw1 ................... [Auth: CHAP, Gateways: 2]
     | o- disks ................................................................... [Disks: 1]
@@ -38,7 +38,7 @@ o- / ...........................................................................
       o- host-groups ........................................................... [Groups : 0]
       o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
     |   o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: CHAP, Disks: 1(2G)]
-    |     o- lun 0 ......................................... [rbd.disk_1(2G), Owner: rh7-gw2]
+    |     o- lun 0 ......................................... [rbd/disk_1(2G), Owner: rh7-gw2]
     o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw2 ................... [Auth: None, Gateways: 2]
       o- disks ................................................................... [Disks: 1]
       | o- rbd/disk_2 .............................................. [Owner: rh7-gw1, Lun: 0]
@@ -48,7 +48,17 @@ o- / ...........................................................................
       o- host-groups ........................................................... [Groups : 0]
       o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
         o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(2G)]
-          o- lun 0 ......................................... [rbd.disk_2(2G), Owner: rh7-gw1]
+          o- lun 0 ......................................... [rbd/disk_2(2G), Owner: rh7-gw1]
+    o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw3 ................... [Auth: None, Gateways: 2]
+      o- disks ................................................................... [Disks: 1]
+      | o- rbd/ec/disk_4 ........................................... [Owner: rh7-gw2, Lun: 0]
+      o- gateways ..................................................... [Up: 2/2, Portals: 2]
+      | o- rh7-gw1 ................................................... [2006:ac81::1103 (UP)]
+      | o- rh7-gw2 ................................................... [2006:ac81::1104 (UP)]
+      o- host-groups ........................................................... [Groups : 0]
+      o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
+        o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(1G)]
+          o- lun 0 ...................................... [rbd/ec/disk_4(1G), Owner: rh7-gw1]
 
 
 
@@ -95,6 +105,30 @@ curl --user admin:admin -d ip_address=2006:ac81::1104 \
 NOTE: please make sure both the IPv4 and IPv6 addresses are in the trusted
 ip list in iscsi-gateway.cfg.
 
+Erasure Pool Support:
+For the erasure pool, you need to specify the "ecpool=<ec_name>" parameter to store the
+data when creating a disk, and the "pool=<name>" will contiue to be a replicated pool,
+which will store the metadata only.
+
+When creating a disk and disk snapshot, for rest api there has a litte different for
+erasure pool.
+
+You need to use "ecdisk" instead of "disk" in URL:"http://.../disk/...":
+
+curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d ecpool=ec -d count=5
+     -X PUT http://192.168.122.69:5000/api/ecdisk/rbd/ec/new0_
+curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d ecpool=ec -d create_image=false
+     -X PUT http://192.168.122.69:5000/api/ecdisk/rbd/ec/new1
+curl --user admin:admin -X GET http://192.168.122.69:5000/api/ecdisk/rbd/ec/new2
+curl --user admin:admin -X DELETE http://192.168.122.69:5000/api/ecdisk/rbd/ec/new3
+
+You need to use "ecdisksnap" instead of "disksnap" in URL:"http://.../disk/...":
+
+curl --user admin:admin -d mode=create
+     -X PUT http://192.168.122.69:5000/api/ecdisksnap/rbd/ec/image/new1
+curl --user admin:admin
+     -X DELETE http://192.168.122.69:5000/api/ecdisksnap/rbd/ec/image/new1
+
 
 ## Installation
 ### Via RPM

diff --git a/ceph_iscsi_config/.device_status.py.swp b/ceph_iscsi_config/.device_status.py.swp
diff --git a/ceph_iscsi_config/client.py b/ceph_iscsi_config/client.py
@@ -98,7 +98,7 @@ def __init__(self, logger, client_iqn, image_list, username, password, mutual_us
             self.error = True
             self.error_msg = err
 
-        # image_list is normally a list of strings (pool/image_name) but
+        # image_list is normally a list of strings (pool[/ecpool]/image_name) but
         # group processing forces a specific lun id allocation to masked disks
         # in this scenario the image list is a tuple
         if image_list:

diff --git a/ceph_iscsi_config/lun.py b/ceph_iscsi_config/lun.py
@@ -1,3 +1,4 @@
+import json
 import rados
 import rbd
 import re
@@ -13,8 +14,9 @@
 from ceph_iscsi_config.backstore import USER_RBD
 from ceph_iscsi_config.utils import (convert_2_bytes, gen_control_string,
                                      valid_size, get_pool_id, ip_addresses,
-                                     get_pools, get_rbd_size, this_host,
-                                     human_size, CephiSCSIError)
+                                     get_pools, get_rbd_size, run_shell_cmd,
+                                     human_size, CephiSCSIError, this_host,
+                                     parse_disk_meta)
 from ceph_iscsi_config.gateway_object import GWObject
 from ceph_iscsi_config.target import GWTarget
 from ceph_iscsi_config.client import GWClient, CHAP
@@ -46,13 +48,14 @@ class RBDDev(object):
         ]
     }
 
-    def __init__(self, image, size, backstore, pool=None):
+    def __init__(self, image, size, backstore, pool=None, ecpool=None):
         self.image = image
         self.size_bytes = convert_2_bytes(size)
         self.backstore = backstore
         if pool is None:
             pool = settings.config.pool
         self.pool = pool
+        self.ecpool = ecpool
         self.pool_id = get_pool_id(pool_name=self.pool)
         self.error = False
         self.error_msg = ''
@@ -74,13 +77,14 @@ def create(self):
                                     self.image,
                                     self.size_bytes,
                                     features=RBDDev.default_features(self.backstore),
-                                    old_format=False)
+                                    old_format=False,
+                                    data_pool=self.ecpool)
 
                 except (rbd.ImageExists, rbd.InvalidArgument) as err:
                     self.error = True
                     self.error_msg = ("Failed to create rbd image {} in "
-                                      "pool {} : {}".format(self.image,
-                                                            self.pool,
+                                      "pool {}, ecpool {} : {}".format(self.image,
+                                                            self.pool, self.ecpool,
                                                             err))
 
     def delete(self):
@@ -289,14 +293,18 @@ class LUN(GWObject):
         USER_RBD: TCMU_SETTINGS
     }
 
-    def __init__(self, logger, pool, image, size, allocating_host,
+    def __init__(self, logger, pool, ecpool, image, size, allocating_host,
                  backstore, backstore_object_name):
         self.logger = logger
         self.image = image
         self.pool = pool
+        self.ecpool = ecpool
         self.pool_id = 0
         self.size_bytes = convert_2_bytes(size)
-        self.config_key = '{}/{}'.format(self.pool, self.image)
+        if ecpool:
+            self.config_key = '{}/{}/{}'.format(pool, ecpool, image)
+        else:
+            self.config_key = '{}/{}'.format(pool, image)
 
         self.allocating_host = allocating_host
         self.backstore = backstore
@@ -351,7 +359,7 @@ def remove_lun(self, preserve_image):
             if self.error:
                 return
 
-        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool, self.ecpool)
 
         if local_gw == self.allocating_host:
             # by using the allocating host we ensure the delete is not
@@ -574,6 +582,38 @@ def activate(self):
             if client_err:
                 raise CephiSCSIError(client_err)
 
+    def _erasure_pool_check(self):
+        if not self.ecpool:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd metadata --format=json".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf))
+        if err:
+            self.logger.error("Cannot get the objectstore type")
+            return err
+        bluestore = False
+        for _osd in json.loads(data):
+            store_type = _osd['osd_objectstore']
+            self.logger.debug(f"pool ({self.pool}) objectstore type is ({store_type})")
+            if store_type == 'bluestore':
+                bluestore = True
+                break
+
+        if not bluestore:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd pool get {pool} allow_ec_overwrites".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf, pool=self.ecpool))
+        if err:
+            self.logger.error(f"Cannot get allow_ec_overwrites from pool ({self.pool})")
+            return err
+        self.logger.debug(f"erasure pool ({self.pool}) allow_ec_overwrites is enabled")
+        return None
+
     def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         """
         Create image and add to LIO and config.
@@ -583,6 +623,10 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         :return: LIO storage object if successful and keep_dev_in_lio=True
                  else None.
         """
+        err = self._erasure_pool_check()
+        if err:
+            return None
+
         self.logger.debug("LUN.allocate starting, listing rbd devices")
         disk_list = RBDDev.rbd_list(pool=self.pool)
         self.logger.debug("rados pool '{}' contains the following - "
@@ -593,7 +637,8 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                           "allocations is {}".format(local_gw,
                                                      self.allocating_host))
 
-        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool,
+                           self.ecpool)
         self.pool_id = rbd_image.pool_id
 
         # if the image required isn't defined, create it!
@@ -703,6 +748,7 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                     disk_attr = {"wwn": wwn,
                                  "image": self.image,
                                  "pool": self.pool,
+                                 "ecpool": self.ecpool,
                                  "allocating_host": self.allocating_host,
                                  "pool_id": rbd_image.pool_id,
                                  "controls": self.controls,
@@ -963,7 +1009,10 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
 
         :param ceph_iscsi_config: Config object
         :param logger: logger object
-        :param image_id: (str) <pool>.<image> format
+        :param pool: (str) pool name
+        :param ecpool: (str) ecpool name
+        :param image: (str) image name
+        :param size: (str) size
         :return: (str) either 'ok' or an error description
         """
 
@@ -993,12 +1042,18 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
 
         config = ceph_iscsi_config.config
 
-        disk_key = "{}/{}".format(kwargs['pool'], kwargs['image'])
+        ecpool = kwargs.get('ecpool', None)
+        if ecpool:
+            disk_key = "{}/{}/{}".format(kwargs['pool'], ecpool, kwargs['image'])
+        else:
+            disk_key = "{}/{}".format(kwargs['pool'], kwargs['image'])
 
         if mode in ['create', 'resize']:
 
             if kwargs['pool'] not in get_pools():
                 return "pool name is invalid"
+            if ecpool and ecpool not in get_pools():
+                return "ecpool name is invalid"
 
         if mode == 'create':
             if kwargs['size'] and not valid_size(kwargs['size']):
@@ -1010,6 +1065,8 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
             disk_regex = re.compile(r"^[a-zA-Z0-9\-_\.]+$")
             if not disk_regex.search(kwargs['pool']):
                 return "Invalid pool name (use alphanumeric, '_', '.', or '-' characters)"
+            if ecpool and not disk_regex.search(ecpool):
+                return "Invalid ecpool name (use alphanumeric, '_', '.', or '-' characters)"
             if not disk_regex.search(kwargs['image']):
                 return "Invalid image name (use alphanumeric, '_', '.', or '-' characters)"
 
@@ -1040,9 +1097,7 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
         if mode in ["resize", "delete", "reconfigure"]:
             # disk must exist in the config
             if disk_key not in config['disks']:
-                return ("rbd {}/{} is not defined to the "
-                        "configuration".format(kwargs['pool'],
-                                               kwargs['image']))
+                return ("rbd {} is not defined to the configuration".format(disk_key))
 
         if mode == 'resize':
 
@@ -1112,7 +1167,7 @@ def _backstore_object_name_exists(disks_config, backstore_object_name_exists):
                     if disk['backstore_object_name'] == backstore_object_name_exists]) > 0
 
     @staticmethod
-    def get_backstore_object_name(pool, image, disks_config):
+    def get_backstore_object_name(pool, ecpool, image, disks_config):
         """
         Determine the backstore storage object name based on the pool name,
         image name, and existing storage object names to avoid conflicts.
@@ -1126,7 +1181,10 @@ def get_backstore_object_name(pool, image, disks_config):
         :param disks_config: disks configuration from `gateway.conf`
         :return: the backstore storage object name to be used
         """
-        base_name = '{}.{}'.format(pool, image)
+        if ecpool:
+            base_name = '{}.{}.{}'.format(pool, ecpool, image)
+        else:
+            base_name = '{}.{}'.format(pool, image)
         candidate = base_name
         counter = 0
         while LUN._backstore_object_name_exists(disks_config, candidate):
@@ -1230,14 +1288,15 @@ def define_luns(logger, config, target):
                                   if disk_key.startswith(pool + '/')]
                     for disk_key in pool_disks:
 
-                        pool, image_name = disk_key.split('/')
+                        pool, ecpool, image_name = parse_disk_meta(disk_key)
+
                         with rbd.Image(ioctx, image_name) as rbd_image:
 
                             disk_config = config.config['disks'][disk_key]
                             backstore = disk_config['backstore']
                             backstore_object_name = disk_config['backstore_object_name']
 
-                            lun = LUN(logger, pool, image_name,
+                            lun = LUN(logger, pool, ecpool, image_name,
                                       rbd_image.size(), local_gw, backstore,
                                       backstore_object_name)
 

diff --git a/ceph_iscsi_config/utils.py b/ceph_iscsi_config/utils.py
@@ -28,6 +28,18 @@ class CephiSCSIInval(CephiSCSIError):
     pass
 
 
+def parse_disk_meta(disk):
+    pool = None
+    ecpool = None
+    image = None
+    try:
+        pool, ecpool, image = disk.split('/')
+    except ValueError:
+        pool, image = disk.split('/')
+        pass
+    return pool, ecpool, image
+
+
 def run_shell_cmd(cmd, stderr=None, shell=True):
     if not stderr:
         stderr = subprocess.STDOUT

diff --git a/gwcli/client.py b/gwcli/client.py
@@ -4,7 +4,7 @@
 
 from ceph_iscsi_config.client import CHAP, GWClient
 import ceph_iscsi_config.settings as settings
-from ceph_iscsi_config.utils import human_size, this_host
+from ceph_iscsi_config.utils import human_size, this_host, parse_disk_meta
 
 from rtslib_fb.utils import normalize_wwn, RTSLibError
 
@@ -540,7 +540,7 @@ def ui_command_disk(self, action='add', disk=None, size=None):
 
                 # a disk given here would be of the form pool.image
                 try:
-                    pool, image = disk.split('/')
+                    pool, ecpool, image = parse_disk_meta(disk)
                 except ValueError:
                     self.logger.error("Invalid format. Use pool_name/disk_name")
                     return