diff --git a/cmd/root.go b/cmd/root.go index 9c91d6f..b0d6f14 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,6 +9,7 @@ import ( var ( configFile string + collection string ) var RootCmd = &cobra.Command{ @@ -22,7 +23,7 @@ var RootCmd = &cobra.Command{ func Execute() { RootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "config YAML file of server") - + RootCmd.PersistentFlags().StringVarP(&collection, "table", "t", "", "migration source ES index or Milvus collection") RootCmd.Execute() } diff --git a/cmd/start.go b/cmd/start.go index 5539c19..78dda51 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -29,7 +29,7 @@ var startCmd = &cobra.Command{ return } }() - err := starter.Start(ctx, configFile, jobId) + err := starter.Start(ctx, configFile, collection, jobId) if err != nil { log.Error("[start migration error]", zap.Error(err)) return diff --git a/core/common/constant.go b/core/common/constant.go index ff42298..7b674de 100644 --- a/core/common/constant.go +++ b/core/common/constant.go @@ -61,3 +61,4 @@ var LOAD_CHECK_BACKLOG_INTERVAL = time.Second * 10 //second const SUB_FILE_SIZE = 1024 * 1024 * 300 const UPSERT = "upsert" +const MILVUS_META_FD = "$meta" diff --git a/core/data/process_info.go b/core/data/process_info.go index 2b1c388..37a8fdb 100644 --- a/core/data/process_info.go +++ b/core/data/process_info.go @@ -107,6 +107,12 @@ func calcDumpProc(p *ProcessHandler) int { } func calcByInsertDataProc(p *ProcessHandler) int { + //fmt.Printf("xxxxxxxx: (%d) ", p.LoadTotalSize) + //数据量很少情况下,count milvus collection可能会返回为0 + if p.LoadTotalSize <= 0 { + return One_Percent + } + loadSize := p.LoadFinishSize.Load() if loadSize == 0 { return Half_Percent diff --git a/core/transform/milvus2x/convert/milvus2x_convert.go b/core/transform/milvus2x/convert/milvus2x_convert.go index 099934d..d5c8a97 100644 --- a/core/transform/milvus2x/convert/milvus2x_convert.go +++ b/core/transform/milvus2x/convert/milvus2x_convert.go @@ -18,6 +18,11 @@ func ToMilvusParam(ctx context.Context, collCfg *milvus2xtype.CollectionCfg, mil if err != nil { return nil, err } + //当source是开启动态列表,并且 target也打开动态列属性,则动态列需要迁移(DynamicField=true) + collCfg.DynamicField = srcCollEntity.Schema.EnableDynamicField && !collCfg.MilvusCfg.CloseDynamicField + + log.Info("milvus2x source collection_schema", zap.Bool("DynamicFieldStatus", collCfg.DynamicField), + zap.String("Collection", collCfg.Collection)) fields, err := ToMilvusFields(srcCollEntity, collCfg) if err != nil { diff --git a/core/type/milvus2xtype/mlv2x_type.go b/core/type/milvus2xtype/mlv2x_type.go index 8186e7e..9a9825c 100644 --- a/core/type/milvus2xtype/mlv2x_type.go +++ b/core/type/milvus2xtype/mlv2x_type.go @@ -14,6 +14,8 @@ type CollectionCfg struct { Rows int64 `json:"rows"` Fields []FieldCfg `json:"fields"` MilvusCfg *milvustype.MilvusCfg `json:"milvus"` + + DynamicField bool //source collection Dynamic Field status, if it open, will sync $meta field data to target collection } type FieldCfg struct { diff --git a/go.mod b/go.mod index a1401ab..1c2a857 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,12 @@ require ( github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 github.com/matoous/go-nanoid/v2 v2.0.0 github.com/milvus-io/milvus-sdk-go v1.1.1 - github.com/milvus-io/milvus-sdk-go/v2 v2.4.1 + github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1 github.com/minio/minio-go/v7 v7.0.66 + github.com/olivere/elastic/v7 v7.0.32 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/satori/go.uuid v1.2.0 github.com/shopspring/decimal v1.3.1 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 @@ -90,7 +92,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.19 // indirect - github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -124,7 +126,7 @@ require ( golang.org/x/tools v0.16.1 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c12f22e..e6d337b 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/congqixia/milvus-sdk-go/v2 v2.0.0-20240819072515-04d118f1639d h1:JkYXo7c1r/ncfAqt32clf196c1mVr3aucCOIVTVjKDs= +github.com/congqixia/milvus-sdk-go/v2 v2.0.0-20240819072515-04d118f1639d/go.mod h1:kQ7SuDWugDhCqEHIvokitjVO9CyATIFa+h1dHEk86Zc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -68,6 +70,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -181,24 +185,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430025921-135167be0694 h1:iub0yx8peGNtnb9n11iuWNmhIhIXw3xfZooIDcrfeU8= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430025921-135167be0694/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4 h1:HtNGcUb52ojnl+zDAZMmbHyVaTdBjzuCnnBHpb675TU= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6 h1:rSkwp5Mg/7KBSUqXcrPBUgTQGZNdvYWEKB+rHo9YJtk= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= github.com/milvus-io/milvus-sdk-go v1.1.1 h1:QseeGBb92T4ny5jSscaWJ+toG74p5zE0kaxFyeskWSE= github.com/milvus-io/milvus-sdk-go v1.1.1/go.mod h1:jyYc5vzEQS9V+ZrSL2h/oeKcYiixvKBV/TrxXbLu01w= -github.com/milvus-io/milvus-sdk-go/v2 v2.3.4 h1:WeZ/QCwpcZVOiaVScuqoKhjuv3DaEAx+jM6U5PJhK+E= -github.com/milvus-io/milvus-sdk-go/v2 v2.3.4/go.mod h1:ubhpNcq6Y25PNl2JabqIlH64yGHAEeo3Y7tgQHXQwnU= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240528112137-8309bdf288fe h1:M8Fp0khgp7gdVne0BfHgy/BqN4a3TxoTV2Iq4SPYWOY= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240528112137-8309bdf288fe/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530061138-979f77e0e810 h1:FVJnBYfQCQKcGJOQbKeYL2BYcp+dv7PRj1u200cAKWQ= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530061138-979f77e0e810/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530093539-f4779551f949 h1:l9++xnkflq2NFvFZ9nQe+VzEFgo1og8mMQ4kUuPUqPc= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530093539-f4779551f949/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1 h1:KhqjmaJE4mSxj1a88XtkGaqgH4duGiHs1sjnvSXkwE0= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.1/go.mod h1:7SJxshlnVhNLksS73tLPtHYY9DiX7lyL43Rv41HCPCw= +github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1 h1:tmPS7djDZ0VrXmgp/UbWKDFwsBuIVTjF+p4jF4Pd3KI= +github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1/go.mod h1:kQ7SuDWugDhCqEHIvokitjVO9CyATIFa+h1dHEk86Zc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= @@ -213,6 +205,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= +github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= @@ -242,6 +236,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -445,8 +441,8 @@ google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQ google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/function.go b/server/function.go index 4fb1ef5..fa4a941 100644 --- a/server/function.go +++ b/server/function.go @@ -121,11 +121,11 @@ func handleStart(c *gin.Context) (interface{}, error) { }() if req.Async { - go starter.Start(log.NewContextWithRequestId(c.Request.Context()), "", jobId) + go starter.Start(log.NewContextWithRequestId(c.Request.Context()), "", "", jobId) return param.NewJobResponse(jobId), nil } - return param.NewJobResponse(jobId), starter.Start(c.Request.Context(), "", jobId) + return param.NewJobResponse(jobId), starter.Start(c.Request.Context(), "", "", jobId) } func handlePanic(_any any, jobId string) { diff --git a/starter/starter.go b/starter/starter.go index f2dd0c6..f4f04ee 100644 --- a/starter/starter.go +++ b/starter/starter.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/zilliztech/milvus-migration/core/cleaner" "github.com/zilliztech/milvus-migration/core/common" + "github.com/zilliztech/milvus-migration/core/config" "github.com/zilliztech/milvus-migration/core/dumper" "github.com/zilliztech/milvus-migration/core/gstore" "github.com/zilliztech/milvus-migration/core/loader" @@ -79,7 +80,11 @@ func Load(ctx context.Context, configFile string, param *param.LoadParam, jobId return nil } -func Start(ctx context.Context, configFile string, jobId string) error { +func Start(ctx context.Context, configFile string, collection string, jobId string) error { + + if collection != "" { + fmt.Printf("Migration CmdParam Collection: %s Start..", collection) + } start := time.Now() @@ -93,6 +98,10 @@ func Start(ctx context.Context, configFile string, jobId string) error { return err } + if collection != "" { + replaceCollectionName(migrCfg, collection) + } + if migrCfg.DumperWorkCfg.WorkMode == string(common.Elasticsearch) { //record: es dump will split many small json file task gstore.InitFileTask(jobId) @@ -118,11 +127,27 @@ func Start(ctx context.Context, configFile string, jobId string) error { } log.LL(ctx).Info("[Cleaner] clean file success!") + if collection != "" { + fmt.Printf("Migration CmdParam Collection: %s Done!", collection) + } + fmt.Printf("Migration Success! Job %s cost=[%f]\n", jobId, time.Since(start).Seconds()) printStartJobMessage(jobId) return nil } +func replaceCollectionName(migrCfg *config.MigrationConfig, collection string) { + if migrCfg.MetaConfig.Milvus2xMeta != nil { + migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].Collection = collection + if migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].MilvusCfg != nil { + migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].MilvusCfg.Collection = collection + } + } + if migrCfg.MetaConfig.EsMeta != nil { + migrCfg.MetaConfig.EsMeta.IdxCfgs[0].MilvusCfg.Collection = collection + } +} + func printStartJobMessage(jobId string) { jobInfo, _ := gstore.GetJobInfo(jobId) val, _ := json.Marshal(&jobInfo) diff --git a/storage/milvus2x/milvus2_3_ver.go b/storage/milvus2x/milvus2_3_ver.go index 3686b83..0aa4529 100644 --- a/storage/milvus2x/milvus2_3_ver.go +++ b/storage/milvus2x/milvus2_3_ver.go @@ -52,10 +52,15 @@ func (milvus23 *Milvus23VerClient) InitIterator(ctx context.Context, collCfg *mi } fieldNames = append(fieldNames, fieldCfg.Name) } + if collCfg.DynamicField { + fieldNames = append(fieldNames, common.MILVUS_META_FD) //把source 动态列也查出来 + } + log.Info("start iterator milvus collection", zap.Any("migration fieldName", fieldNames)) log.Info("start iterator milvus collection", zap.Any("migration milvusCfg", collCfg.MilvusCfg)) log.Info("start iterator milvus collection", zap.Any("migration fields", collCfg.Fields)) iteratorParam := client.NewQueryIteratorOption(collCfg.Collection).WithBatchSize(batchSize).WithExpr(common.EMPTY).WithOutputFields(fieldNames...) + //iteratorParam := client.NewQueryIteratorOption(collCfg.Collection).WithBatchSize(batchSize).WithExpr(common.EMPTY).WithOutputFields("*") iterator, err := milvus23._milvus.QueryIterator(ctx, iteratorParam) if err != nil { return err @@ -80,9 +85,19 @@ func (milvus23 *Milvus23VerClient) IterateNext(ctx context.Context) (*Milvus2xDa } columns := make([]entity.Column, 0, len(rs)) for _, col := range rs { - columns = append(columns, col) + if col.Name() == common.MILVUS_META_FD { + data := col.(*entity.ColumnJSONBytes).Data() + dynamicCol := entity.NewColumnJSONBytes(common.EMPTY, data).WithIsDynamic(true) + columns = append(columns, dynamicCol) + log.Info("[Milvus2x] iterateNext data ======> $meta") + } else { + columns = append(columns, col) + } if common.DEBUG { log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("colLen", col.Len())) + log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("FieldData", col.FieldData().FieldName)) + log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("IsDynamic", col.FieldData().IsDynamic)) + log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("FieldDataVal", col.FieldData().String())) } } if common.DEBUG { diff --git a/testfiles/milvus2x/batch_collection_migration.sh b/testfiles/milvus2x/batch_collection_migration.sh new file mode 100755 index 0000000..4f2b658 --- /dev/null +++ b/testfiles/milvus2x/batch_collection_migration.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +collections=("collection1" "collection2" "collection3") + +for collection in "${collections[@]}"; do + echo "BatchMigration==> $collection" + ./milvus-migration start -t="$collection" -c=/{YourConfigPath}/migration.yml +done + +# how to execute? +#1. chmod +x batch_collection_migration.sh +#2. ./batch_collection_migration.sh \ No newline at end of file