fix: cleanup volume-snapshot backup failure (#4783)

cmd/backup-manager/app/clean/clean.go

@@ -16,6 +16,7 @@ package clean
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -75,16 +76,19 @@ func (bo *Options) deleteSnapshotsAndBackupMeta(ctx context.Context, backup *v1a
 	//rclone copy remote:/bukect/backup/backupmeta /backupmeta
 	opts := util.GetOptions(backup.Spec.StorageProvider)
 	if err := bo.copyRemoteBackupMetaToLocal(ctx, backup.Status.BackupPath, opts); err != nil {
-		klog.Errorf("rclone copy remote backupmeta to local failure.")
-		return err
+		klog.Warningf("rclone copy remote backupmeta to local failure, err: %s. it possible that bucket or backup folder is deleted already. a mannual check is require", err)
+		return nil
 	}
 	defer func() {
 		_ = os.Remove(metaFile)
 	}()
 
 	contents, err := os.ReadFile(metaFile)
-	if err != nil {
-		klog.Errorf("read metadata file %s failed, err: %s", metaFile, err)
+	if errors.Is(err, os.ErrNotExist) {
+		klog.Warningf("read metadata file %s failed, err: %s, a mannual check or delete aciton require.", metaFile, err)
+		return nil
+	} else if err != nil { // will retry it
+		klog.Warningf("read metadata file %s failed, err: %s", metaFile, err)
 		return err
 	}
 
@@ -96,6 +100,7 @@ func (bo *Options) deleteSnapshotsAndBackupMeta(ctx context.Context, backup *v1a
 
 	//2. delete the snapshot
 	if err = bo.deleteVolumeSnapshots(metaInfo); err != nil {
+		klog.Errorf("delete volume snapshot failure, a mannual check or delete aciton require.")
 		return err
 	}