Use openapi generator for export and add export GCS/Azure/Parquet (#218)

tidbcloud · Aug 29, 2024 · ab94f76 · ab94f76
1 parent 73399bf
commit ab94f76
Show file tree

Hide file tree

Showing 87 changed files with 11,178 additions and 5,829 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ dist/
 
 .tidbcloud-cli.toml
 .CHANGELOG.md
+node_modules/
diff --git a/Makefile b/Makefile
@@ -21,7 +21,7 @@ setup: deps devtools ## Set up dev env
 .PHONY: generate-mocks
 generate-mocks: ## Generate mock objects
 	@echo "==> Generating mock objects"
-	go install github.com/vektra/mockery/v2@latest
+	go install github.com/vektra/mockery/v2@v2.43.0
 	mockery --name TiDBCloudClient --recursive --output=internal/mock --outpkg mock --filename api_client.go
 	mockery --name EventsSender --recursive --output=internal/mock --outpkg mock --filename sender.go
 	mockery --name Uploader --recursive --output=internal/mock --outpkg mock --filename uploader.go
@@ -44,7 +44,7 @@ addcopy: ## Add copyright to all files
 	@scripts/add-copy.sh
 
 .PHONY: generate-v1beta1-client
-generate-v1beta1-client: ## Generate v1beta1 client
+generate-v1beta1-client: install-openapi-generator## Generate v1beta1 client
 	go install github.com/go-swagger/go-swagger/cmd/swagger@latest
 	@echo "==> Generating serverless branch client"
 	swagger generate client -f pkg/tidbcloud/v1beta1/branch/branch.swagger.json -A tidbcloud-serverless -t pkg/tidbcloud/v1beta1/branch
@@ -55,10 +55,15 @@ generate-v1beta1-client: ## Generate v1beta1 client
 	@echo "==> Generating serverless import client"
 	swagger generate client -f pkg/tidbcloud/v1beta1/serverless_import/import.swagger.json -A tidbcloud-serverless -t pkg/tidbcloud/v1beta1/serverless_import
 	@echo "==> Generating serverless export client"
-	swagger generate client -f pkg/tidbcloud/v1beta1/serverless_export/export.swagger.json -A tidbcloud-serverless -t pkg/tidbcloud/v1beta1/serverless_export
+	rm -rf pkg/tidbcloud/v1beta1/serverless/export
+	cd tools/openapi-generator && npx openapi-generator-cli generate --additional-properties=withGoMod=false,enumClassPrefix=true --global-property=apiTests=false,apiDocs=false,modelDocs=fasle,modelTests=false -i ../../pkg/tidbcloud/v1beta1/serverless/export.swagger.json -g go -o ../../pkg/tidbcloud/v1beta1/serverless/export --package-name export
 	@echo "==> Generating iam client"
 	swagger generate client -f pkg/tidbcloud/v1beta1/iam/iam.swagger.json -A tidbcloud-serverless -t pkg/tidbcloud/v1beta1/iam
+	go fmt ./pkg/...
 
+.PHONY: install-openapi-generator
+install-openapi-generator:
+	cd tools/openapi-generator && npm install
 
 .PHONY: fmt
 fmt: ## Format changed go
@@ -88,4 +93,4 @@ help:
 .PHONY: generate-docs
 generate-docs: ## Generate mock objects
 	@echo "==> Generating docs"
-	go run gen_doc.go
+	go run gen_doc.go
diff --git a/docs/generate_doc/ticloud_serverless_export_create.md b/docs/generate_doc/ticloud_serverless_export_create.md
@@ -15,7 +15,7 @@ ticloud serverless export create [flags]
   Export all data with local type in non-interactive mode:
   $ ticloud serverless export create -c <cluster-id>
 
-  Export all data with s3 type in non-interactive mode:
+  Export all data with S3 type in non-interactive mode:
   $ ticloud serverless export create -c <cluster-id> --target-type S3 --s3.uri <s3-uri> --s3.access-key-id <access-key-id> --s3.secret-access-key <secret-access-key>
 
   Export all data and customize csv format in non-interactive mode:
@@ -31,22 +31,28 @@ ticloud serverless export create [flags]
 ### Options
 
 ```
-  -c, --cluster-id string             The ID of the cluster, in which the export will be created.
-      --compression string            The compression algorithm of the export file. One of ["GZIP" "SNAPPY" "ZSTD" "NONE"]. (default "GZIP")
-      --csv.delimiter string          Delimiter of string type variables in CSV files. (default "\"")
-      --csv.null-value string         Representation of null values in CSV files. (default "\\N")
-      --csv.separator string          Separator of each value in CSV files. (default ",")
-      --csv.skip-header               Export CSV files of the tables without header.
-      --file-type string              The export file type. One of ["CSV" "SQL"]. (default "SQL")
-      --filter strings                Specify the exported table(s) with table filter patterns. See https://docs.pingcap.com/tidb/stable/table-filter to learn table filter.
-      --force                         Create without confirmation. You need to confirm when you want to export the whole cluster in non-interactive mode.
-  -h, --help                          help for create
-      --s3.access-key-id string       The access key ID of the S3. Required when target type is S3.
-      --s3.secret-access-key string   The secret access key of the S3. Required when target type is S3.
-      --s3.uri string                 The s3 uri in s3://<bucket>/<path> format. Required when target type is S3.
-      --sql string                    Filter the exported data with SQL SELECT statement.
-      --target-type string            The export target. One of ["LOCAL" "S3"]. (default "LOCAL")
-      --where string                  Filter the exported table(s) with the where condition.
+      --azblob.sas-token string          The SAS token of Azure Blob.
+      --azblob.uri string                The Azure Blob URI in azure://<account>.blob.core.windows.net/<container>/<path> format. Required when target type is AZURE_BLOB.
+  -c, --cluster-id string                The ID of the cluster, in which the export will be created.
+      --compression string               The compression algorithm of the export file. One of ["GZIP" "SNAPPY" "ZSTD" "NONE"]. (default "GZIP")
+      --csv.delimiter string             Delimiter of string type variables in CSV files. (default "\"")
+      --csv.null-value string            Representation of null values in CSV files. (default "\\N")
+      --csv.separator string             Separator of each value in CSV files. (default ",")
+      --csv.skip-header                  Export CSV files of the tables without header.
+      --file-type string                 The export file type. One of ["CSV" "SQL" "PARQUET"]. (default "SQL")
+      --filter strings                   Specify the exported table(s) with table filter patterns. See https://docs.pingcap.com/tidb/stable/table-filter to learn table filter.
+      --force                            Create without confirmation. You need to confirm when you want to export the whole cluster in non-interactive mode.
+      --gcs.service-account-key string   The base64 encoded service account key of GCS.
+      --gcs.uri string                   The GCS URI in gcs://<bucket>/<path> format. Required when target type is GCS.
+  -h, --help                             help for create
+      --parquet.compression string       The parquet compression algorithm. One of ["GZIP" "SNAPPY" "ZSTD" "NONE"]. (default "ZSTD")
+      --s3.access-key-id string          The access key ID of the S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key].
+      --s3.role-arn string               The role arn of the S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key].
+      --s3.secret-access-key string      The secret access key of the S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key].
+      --s3.uri string                    The S3 URI in s3://<bucket>/<path> format. Required when target type is S3.
+      --sql string                       Filter the exported data with SQL SELECT statement.
+      --target-type string               The export target. One of ["LOCAL" "S3" "GCS" "AZURE_BLOB"]. (default "LOCAL")
+      --where string                     Filter the exported table(s) with the where condition.
 ```
 
 ### Options inherited from parent commands

diff --git a/internal/cli/serverless/export/cancel.go b/internal/cli/serverless/export/cancel.go
@@ -28,7 +28,6 @@ import (
 	"tidbcloud-cli/internal/flag"
 	"tidbcloud-cli/internal/service/cloud"
 	"tidbcloud-cli/internal/util"
-	exportApi "tidbcloud-cli/pkg/tidbcloud/v1beta1/serverless_export/client/export_service"
 )
 
 const confirmed = "yes"
@@ -152,9 +151,7 @@ func CancelCmd(h *internal.Helper) *cobra.Command {
 				}
 			}
 
-			params := exportApi.NewExportServiceCancelExportParams().
-				WithClusterID(clusterID).WithExportID(exportID).WithContext(ctx)
-			_, err = d.CancelExport(params)
+			_, err = d.CancelExport(ctx, clusterID, exportID)
 			if err != nil {
 				return errors.Trace(err)
 			}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,3 +11,4 @@ dist/

		.tidbcloud-cli.toml
		.CHANGELOG.md
		node_modules/