increased backoff for attach/detach

reflect.StructField requires atleast go 1.17 or above

Signed-off-by: Niclas Schad <[email protected]>

add exponential backoff for volume readiness (RPC: CreateVolume) (#7)

Signed-off-by: Niclas Schad <[email protected]>

.snyk

@@ -0,0 +1,10 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+# ignores vulnerabilities until expiry date; change duration by modifying expiry date
+ignore:
+  SNYK-GOLANG-GITHUBCOMEMICKLEIGORESTFUL-2435653:
+    - '*':
+        reason: We do not use CORS
+        expires: 2022-06-12T09:26:24.310Z
+        created: 2022-05-13T09:26:24.313Z
+version: v1.25.0
+patch: {}

Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1.3-labs
+ARG GO_VERSION=1.17
+
+FROM golang:${GO_VERSION} AS base
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOARCH=amd64
+
+WORKDIR /src
+COPY go.* .
+
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+FROM base AS build
+
+RUN --mount=target=. \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -ldflags="-w -s" -o /app/main cmd/cinder-csi-plugin/main.go
+
+FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3
+
+LABEL name="cinder-csi-plugin" \
+      license="Apache Version 2.0" \
+      maintainers="Kubernetes Authors" \
+      description="Cinder CSI Plugin" \
+      architecture=amd64 \
+      distribution-scope="public" \
+      summary="Cinder CSI Plugin" \
+      help="none"
+
+RUN clean-install ca-certificates e2fsprogs mount xfsprogs udev
+
+COPY --from=build /app/main /bin/controller
+ENTRYPOINT ["/bin/controller"]

Makefile

@@ -31,8 +31,8 @@ TEMP_DIR	:=$(shell mktemp -d)
 TAR_FILE	?= rootfs.tar
 
 GOOS		?= $(shell go env GOOS)
-GIT_VERSION     := $(shell git describe --dirty --tags --match='v*')
-VERSION         ?= $(GIT_VERSION)
+VERSION		?= $(shell git describe --exact-match > /dev/null || \
+			   git describe --tags --always --abbrev=6)
 ALPINE_ARCH	:=
 DEBIAN_ARCH	:=
 QEMUARCH	:=
@@ -42,7 +42,7 @@ GOFLAGS		:=
 TAGS		:=
 LDFLAGS		:= "-w -s -X 'k8s.io/component-base/version.gitVersion=$(VERSION)'"
 GOX_LDFLAGS	:= $(shell echo "$(LDFLAGS) -extldflags \"-static\"")
-REGISTRY	?= k8scloudprovider
+REGISTRY	?= reg.infra.ske.eu01.stackit.cloud/stackitcloud
 IMAGE_OS	?= linux
 IMAGE_NAMES	?= openstack-cloud-controller-manager \
 				cinder-csi-plugin \
@@ -122,9 +122,9 @@ manila-csi-plugin: work $(SOURCES)
 # Remove this individual go build target, once we remove
 # image-controller-manager below.
 openstack-cloud-controller-manager: work $(SOURCES)
-	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \
+	CGO_ENABLED=0 GOOS=$(GOOS) go build \
 		-ldflags $(LDFLAGS) \
-		-o openstack-cloud-controller-manager-$(ARCH) \
+		-o openstack-cloud-controller-manager \
 		cmd/openstack-cloud-controller-manager/main.go
 
 # Remove individual image builder once we migrate openlab-zuul-jobs

go.mod

@@ -120,7 +120,7 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/term v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
@@ -151,6 +151,7 @@ require (
 
 replace (
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9
+	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.11.1
 	google.golang.org/grpc v1.34.0 => google.golang.org/grpc v1.29.0
 	k8s.io/api => k8s.io/api v0.24.0
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.0

pkg/csi/cinder/controllerserver.go

@@ -18,7 +18,9 @@ package cinder
 
 import (
 	"fmt"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"strconv"
+	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/golang/protobuf/ptypes"
@@ -97,12 +99,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		if volSizeGB != volumes[0].Size {
 			return nil, status.Error(codes.AlreadyExists, "Volume Already exists with same name and different capacity")
 		}
+
+		if volumes[0].Status != openstack.VolumeAvailableStatus {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("Volume %s is not in available state", volumes[0].ID))
+		}
+
 		klog.V(4).Infof("Volume %s already exists in Availability Zone: %s of size %d GiB", volumes[0].ID, volumes[0].AvailabilityZone, volumes[0].Size)
 		return getCreateVolumeResponse(&volumes[0], ignoreVolumeAZ, req.GetAccessibilityRequirements()), nil
 	} else if len(volumes) > 1 {
 		klog.V(3).Infof("found multiple existing volumes with selected name (%s) during create", volName)
 		return nil, status.Error(codes.Internal, "Multiple volumes reported by Cinder with same name")
-
 	}
 
 	// Volume Create
@@ -140,11 +146,21 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	}
 
 	vol, err := cloud.CreateVolume(volName, volSizeGB, volType, volAvailability, snapshotID, sourcevolID, &properties)
-
 	if err != nil {
 		klog.Errorf("Failed to CreateVolume: %v", err)
 		return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed with error %v", err))
+	}
 
+	targetStatus := []string{openstack.VolumeAvailableStatus}
+	err = cloud.WaitVolumeTargetStatusWithCustomBackoff(vol.ID, targetStatus,
+		&wait.Backoff{
+			Duration: 20 * time.Second,
+			Steps:    5,
+			Factor:   1.28,
+		})
+	if err != nil {
+		klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", vol.ID, err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume Volume %s failed getting available in time: %v", vol.ID, err))
 	}
 
 	klog.V(4).Infof("CreateVolume: Successfully created volume %s in Availability Zone: %s of size %d GiB", vol.ID, vol.AvailabilityZone, vol.Size)

pkg/csi/cinder/openstack/openstack.go

@@ -18,6 +18,7 @@ package openstack
 
 import (
 	"fmt"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"os"
 
 	"github.com/gophercloud/gophercloud"
@@ -49,6 +50,7 @@ type IOpenStack interface {
 	DetachVolume(instanceID, volumeID string) error
 	WaitDiskDetached(instanceID string, volumeID string) error
 	WaitVolumeTargetStatus(volumeID string, tStatus []string) error
+	WaitVolumeTargetStatusWithCustomBackoff(volumeID string, tStatus []string, backoff *wait.Backoff) error
 	GetAttachmentDiskPath(instanceID, volumeID string) (string, error)
 	GetVolume(volumeID string) (*volumes.Volume, error)
 	GetVolumesByName(name string) ([]volumes.Volume, error)

pkg/csi/cinder/openstack/openstack_volumes.go

@@ -38,12 +38,12 @@ const (
 	operationFinishInitDelay = 1 * time.Second
 	operationFinishFactor    = 1.1
 	operationFinishSteps     = 10
-	diskAttachInitDelay      = 1 * time.Second
-	diskAttachFactor         = 1.2
-	diskAttachSteps          = 15
-	diskDetachInitDelay      = 1 * time.Second
-	diskDetachFactor         = 1.2
-	diskDetachSteps          = 13
+	diskAttachInitDelay      = 7 * time.Second
+	diskAttachFactor         = 1.4
+	diskAttachSteps          = 10
+	diskDetachInitDelay      = 7 * time.Second
+	diskDetachFactor         = 1.4
+	diskDetachSteps          = 10
 	volumeDescription        = "Created by OpenStack Cinder CSI driver"
 )
 
@@ -195,6 +195,12 @@ func (os *OpenStack) AttachVolume(instanceID, volumeID string) (string, error) {
 		return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
 	}
 
+	//redundant waitDiskAttached, workaround for raise condition in backend
+	err = os.WaitDiskAttached(instanceID, volumeID)
+	if err != nil {
+		return "", err
+	}
+
 	return volume.ID, nil
 }
 
@@ -217,7 +223,7 @@ func (os *OpenStack) WaitDiskAttached(instanceID string, volumeID string) error
 	})
 
 	if err == wait.ErrWaitTimeout {
-		err = fmt.Errorf("Volume %q failed to be attached within the alloted time", volumeID)
+		err = fmt.Errorf("Volume %q failed to be attached within the allowed time", volumeID)
 	}
 
 	return err
@@ -256,6 +262,33 @@ func (os *OpenStack) WaitVolumeTargetStatus(volumeID string, tStatus []string) e
 	return waitErr
 }
 
+//WaitVolumeTargetStatusWithCustomBackoff waits for volume to be in target state with custom backoff
+func (os *OpenStack) WaitVolumeTargetStatusWithCustomBackoff(volumeID string, tStatus []string, backoff *wait.Backoff) error {
+	waitErr := wait.ExponentialBackoff(*backoff, func() (bool, error) {
+		vol, err := os.GetVolume(volumeID)
+		if err != nil {
+			return false, err
+		}
+		for _, t := range tStatus {
+			if vol.Status == t {
+				return true, nil
+			}
+		}
+		for _, eState := range volumeErrorStates {
+			if vol.Status == eState {
+				return false, fmt.Errorf("Volume is in Error State : %s", vol.Status)
+			}
+		}
+		return false, nil
+	})
+
+	if waitErr == wait.ErrWaitTimeout {
+		waitErr = fmt.Errorf("Timeout on waiting for volume %s status to be in %v", volumeID, tStatus)
+	}
+
+	return waitErr
+}
+
 // DetachVolume detaches given cinder volume from the compute
 func (os *OpenStack) DetachVolume(instanceID, volumeID string) error {
 	volume, err := os.GetVolume(volumeID)