Perform object storage test at startup (#5281)

Signed-off-by: Changxin Miao <[email protected]>

cmd/flags.go

@@ -145,6 +145,11 @@ func storageFlags() []cli.Flag {
 			Name:  "download-limit",
 			Usage: "bandwidth limit for download in Mbps",
 		},
+		&cli.BoolFlag{
+			Name: "check-storage",
+			// AK/SK should have been checked before creating volume, here checks client access to the storage
+			Usage: "test storage before mounting to expose access issues early",
+		},
 	})
 }
 

cmd/format.go

@@ -341,7 +341,7 @@ func doTesting(store object.ObjectStorage, key string, data []byte) error {
 		return err
 	}
 	if !bytes.Equal(data, data2) {
-		return fmt.Errorf("Read wrong data")
+		return fmt.Errorf("read wrong data: expected %x, got %x", data, data2)
 	}
 	err = store.Delete(key)
 	if err != nil {
@@ -362,6 +362,7 @@ func test(store object.ObjectStorage) error {
 		if err == nil {
 			break
 		}
+		logger.Warnf("Test storage %s failed: %s, tries: #%d", store, err, i+1)
 		time.Sleep(time.Second * time.Duration(i*3+1))
 	}
 	if err == nil {

cmd/mount.go

@@ -644,6 +644,16 @@ func mount(c *cli.Context) error {
 			}
 		}
 		if blob != nil {
+			// test storage at startup to fail fast instead of throwing EIO in the middle of user's workload
+			if c.Bool("check-storage") {
+				start := time.Now()
+				if err = test(blob); err != nil {
+					logger.Errorf("Object storage test failed: %s", err)
+					return err
+				} else {
+					logger.Infof("Object storage test passed in %s", time.Since(start))
+				}
+			}
 			object.Shutdown(blob)
 		}
 		var foreground bool