Do some refine (#206)

* Remove compete cmd Signed-off-by: wayblink <[email protected]> * Update README Signed-off-by: wayblink <[email protected]> * Refine some commands Signed-off-by: wayblink <[email protected]> --------- Signed-off-by: wayblink <[email protected]>
zilliztech · Sep 18, 2023 · db1140d · db1140d
1 parent 6bcf899
commit db1140d
Show file tree

Hide file tree

Showing 10 changed files with 248 additions and 198 deletions.
diff --git a/README.md b/README.md
@@ -2,7 +2,16 @@
 
 Milvus-Backup is a tool that allows users to backup and restore Milvus data. This tool can be utilized either through the command line or an API server.
 
-In order to use Milvus-Backup effectively, access to Milvus proxy and Minio cluster is required. Configuration settings related to this access can be edited in `configs/backup.yaml`.
+The Milvus-backup process has negligible impact on the performance of Milvus. Milvus cluster is fully functional and can operate normally while backup and restoration are in progress.
+
+## Compatibility
+|      Milvus      |  Milvus-backup   |
+|:----------------:|:----------------:|
+| v2.2.9 and above | v0.3.0 and above |
+| v2.2.0 to v2.2.8 | v0.1.0 to v0.2.2 |
+
+## Config
+In order to use Milvus-Backup, access to Milvus proxy and Minio cluster is required. Configuration settings related to this access can be edited in `configs/backup.yaml`.
 
 > **Note**
 >
@@ -14,15 +23,6 @@ In order to use Milvus-Backup effectively, access to Milvus proxy and Minio clus
 > |bucketName|a-bucket|milvus-bucket|
 > |rootPath|files|file|
 
-The Milvus-backup process has negligible impact on the performance of Milvus. Milvus cluster is fully functional and can operate normally while backup and restoration are in progress.
-
-## Compatibility
-|      Milvus      |  Milvus-backup   |
-|:----------------:|:----------------:|
-| v2.2.9 and above | v0.3.0 and above |
-| v2.2.0 to v2.2.8 | v0.1.0 to v0.2.2 |
-
-
 ## Development
 
 ### Build
@@ -144,14 +144,14 @@ curl --location --request GET 'http://localhost:8080/api/v1/get_restore?id=test_
 Milvus-backup establish CLI based on cobra. Use the following command to see the usage.
 
 ```
-milvus-backup is a backup tool for milvus.
+milvus-backup is a backup&restore tool for milvus.
 
 Usage:
   milvus-backup [flags]
   milvus-backup [command]
 
 Available Commands:
-  completion  Generate the autocompletion script for the specified shell
+  check       check if the connects is right.
   create      create subcommand create a backup.
   delete      delete subcommand delete backup by name.
   get         get subcommand get backup by name.
@@ -161,7 +161,8 @@ Available Commands:
   server      server subcommand start milvus-backup RESTAPI server.
 
 Flags:
-  -h, --help   help for milvus-backup
+      --config string   config YAML file of milvus (default "backup.yaml")
+  -h, --help            help for milvus-backup
 
 Use "milvus-backup [command] --help" for more information about a command.
 ```
@@ -170,6 +171,26 @@ Use "milvus-backup [command] --help" for more information about a command.
 
 To try this demo, you should have a functional Milvus server installed and have pymilvus library installed.
 
+Step 0: Check the connections
+
+First of all, we can use `check` command to check whether connections to milvus and storage is normal:
+
+```
+./milvus-backup check
+```
+
+normal output:
+
+```shell
+Succeed to connect to milvus and storage.
+Milvus version: v2.3
+Storage:
+milvus-bucket: a-bucket
+milvus-rootpath: files
+backup-bucket: a-bucket
+backup-rootpath: backup
+```
+
 Step 1: Prepare the Data
 
 Create a collection in Milvus called `hello_milvus` and insert some data using the following command:

diff --git a/cmd/get.go b/cmd/get.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"github.com/spf13/cobra"
@@ -12,6 +13,7 @@ import (
 
 var (
 	getBackName string
+	getDetail   bool
 )
 
 var getBackupCmd = &cobra.Command{
@@ -28,15 +30,19 @@ var getBackupCmd = &cobra.Command{
 		backupContext := core.CreateBackupContext(context, params)
 
 		resp := backupContext.GetBackup(context, &backuppb.GetBackupRequest{
-			BackupName: getBackName,
+			BackupName:    getBackName,
+			WithoutDetail: !getDetail,
 		})
 
-		fmt.Println(resp.GetCode(), "\n", resp.GetMsg())
+		output, _ := json.MarshalIndent(resp.GetData(), "", "    ")
+		fmt.Println(string(output))
+		fmt.Println(resp.GetCode())
 	},
 }
 
 func init() {
 	getBackupCmd.Flags().StringVarP(&getBackName, "name", "n", "", "get backup with this name")
+	getBackupCmd.Flags().BoolVarP(&getDetail, "detail", "d", false, "get complete backup info")
 
 	rootCmd.AddCommand(getBackupCmd)
 }
diff --git a/cmd/restore.go b/cmd/restore.go
@@ -44,13 +44,18 @@ var restoreBackupCmd = &cobra.Command{
 			collectionNameArr = strings.Split(restoreCollectionNames, ",")
 		}
 
-		var renameMap map[string]string
-		if renameCollectionNames == "" {
-			renameMap = map[string]string{}
-		} else {
+		renameMap := make(map[string]string, 0)
+		if renameCollectionNames != "" {
+			fmt.Println("rename: " + renameCollectionNames)
 			renameArr := strings.Split(renameCollectionNames, ",")
-			if len(renameArr) != len(collectionNameArr) {
-				fmt.Errorf("collection_names and renames num dismatch, Forbid to restore")
+			for _, rename := range renameArr {
+				if strings.Contains(rename, ":") {
+					splits := strings.Split(rename, ":")
+					renameMap[splits[0]] = splits[1]
+				} else {
+					fmt.Println("illegal rename parameter")
+					return
+				}
 			}
 		}
 
@@ -83,9 +88,9 @@ func init() {
 	restoreBackupCmd.Flags().StringVarP(&restoreBackupName, "name", "n", "", "backup name to restore")
 	restoreBackupCmd.Flags().StringVarP(&restoreCollectionNames, "collections", "c", "", "collectionNames to restore")
 	restoreBackupCmd.Flags().StringVarP(&renameSuffix, "suffix", "s", "", "add a suffix to collection name to restore")
-	restoreBackupCmd.Flags().StringVarP(&renameCollectionNames, "rename", "r", "", "rename collections to new names")
+	restoreBackupCmd.Flags().StringVarP(&renameCollectionNames, "rename", "r", "", "rename collections to new names, format: db1.collection1:db2.collection1_new,db1.collection2:db2.collection2_new")
 	restoreBackupCmd.Flags().StringVarP(&restoreDatabases, "databases", "d", "", "databases to restore, if not set, restore all databases")
-	restoreBackupCmd.Flags().StringVarP(&restoreDatabaseCollections, "database_collections", "f", "", "databases and collections to restore, json format: {\"db1\":[\"c1\", \"c2\"],\"db2\":[]}")
+	restoreBackupCmd.Flags().StringVarP(&restoreDatabaseCollections, "database_collections", "a", "", "databases and collections to restore, json format: {\"db1\":[\"c1\", \"c2\"],\"db2\":[]}")
 
 	rootCmd.AddCommand(restoreBackupCmd)
 }
diff --git a/cmd/root.go b/cmd/root.go
@@ -13,16 +13,16 @@ var (
 
 var rootCmd = &cobra.Command{
 	Use:   "milvus-backup",
-	Short: "milvus-backup is a backup tool for milvus.",
-	Long:  `milvus-backup is a backup tool for milvus.`,
+	Short: "milvus-backup is a backup&restore tool for milvus.",
+	Long:  `milvus-backup is a backup&restore tool for milvus.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		Error(cmd, args, errors.New("unrecognized command"))
 	},
 }
 
 func Execute() {
 	rootCmd.PersistentFlags().StringVarP(&config, "config", "", "backup.yaml", "config YAML file of milvus")
-
+	rootCmd.CompletionOptions.DisableDefaultCmd = true
 	rootCmd.Execute()
 }
 

diff --git a/configs/backup.yaml b/configs/backup.yaml
@@ -1,7 +1,7 @@
 # Configures the system log output.
 log:
   level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
-  console: true
+  console: true # whether print log to console
   file:
     rootPath: "logs/backup.log"
 

diff --git a/core/backup_context.go b/core/backup_context.go
@@ -197,7 +197,9 @@ func (b *BackupContext) GetBackup(ctx context.Context, request *backuppb.GetBack
 			resp.Code = backuppb.ResponseCode_Fail
 			resp.Msg = err.Error()
 		}
-	} else if request.GetBackupId() == "" && request.GetBackupName() == "" {
+	}
+
+	if request.GetBackupId() == "" && request.GetBackupName() == "" {
 		resp.Code = backuppb.ResponseCode_Parameter_Error
 		resp.Msg = "empty backup name and backup id"
 	} else if request.GetBackupId() != "" {
@@ -245,6 +247,10 @@ func (b *BackupContext) GetBackup(ctx context.Context, request *backuppb.GetBack
 		}
 	}
 
+	if request.WithoutDetail {
+		resp = SimpleBackupResponse(resp)
+	}
+
 	log.Info("finish GetBackupRequest",
 		zap.String("requestId", request.GetRequestId()),
 		zap.String("backupName", request.GetBackupName()),

diff --git a/core/backup_context_test.go b/core/backup_context_test.go
@@ -62,7 +62,7 @@ func TestGetBackup(t *testing.T) {
 	backupContext := CreateBackupContext(context, params)
 
 	backup := backupContext.GetBackup(context, &backuppb.GetBackupRequest{
-		BackupName: "test_backup",
+		BackupName: "mybackup",
 	})
 	assert.Equal(t, backup.GetCode(), backuppb.ResponseCode_Success)
 }

diff --git a/core/backup_impl_restore_backup.go b/core/backup_impl_restore_backup.go
@@ -482,10 +482,10 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
 			err = b.executeBulkInsert(ctx, targetCollectionName, partitionBackup.GetPartitionName(), realFiles, int64(task.GetCollBackup().BackupTimestamp))
 			if err != nil {
 				log.Error("fail to bulk insert to partition",
-					zap.Error(err),
 					zap.String("backupCollectionName", task.GetCollBackup().GetCollectionName()),
 					zap.String("targetCollectionName", targetCollectionName),
-					zap.String("partition", partitionBackup.GetPartitionName()))
+					zap.String("partition", partitionBackup.GetPartitionName()),
+					zap.Error(err))
 				return err
 			}
 			return nil

diff --git a/core/proto/backup.proto b/core/proto/backup.proto
@@ -176,6 +176,8 @@ message GetBackupRequest {
   string bucket_name = 4;
   // if bucket_name and path is set. will override bucket/path in config.
   string path = 5;
+  // if true, return simple response without too much detail to display
+  bool without_detail = 6;
 }
 
 message ListBackupsRequest {