diff --git a/batch.go b/batch.go index 05089dd..ea10c19 100644 --- a/batch.go +++ b/batch.go @@ -13,7 +13,7 @@ type BatchRequestHeaders struct { XCosAppid int `header:"x-cos-appid" xml:"-" url:"-"` ContentLength string `header:"Content-Length,omitempty" xml:"-" url:"-"` ContentType string `header:"Content-Type,omitempty" xml:"-" url:"-"` - Headers *http.Header `header:"-" xml:"-", url:"-"` + Headers *http.Header `header:"-" xml:"-" url:"-"` } // BatchProgressSummary @@ -244,7 +244,7 @@ func (s *BatchService) UpdateJobPriority(ctx context.Context, opt *BatchUpdatePr type BatchUpdateStatusOptions struct { JobId string `header:"-" url:"-" xml:"-"` RequestedJobStatus string `url:"requestedJobStatus" header:"-" xml:"-"` - StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-", xml:"-"` + StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-" xml:"-"` } type BatchUpdateStatusResult struct { XMLName xml.Name `xml:"UpdateJobStatusResult"` diff --git a/bucket_test.go b/bucket_test.go index 16c55a2..ba9ae7d 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -152,14 +152,14 @@ func TestBucketService_Head(t *testing.T) { } func TestBucketService_GetObjectVersions(t *testing.T) { - setup() - defer teardown() + setup() + defer teardown() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, http.MethodGet) - w.WriteHeader(http.StatusOK) - vs := values{ - "versions": "", + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, http.MethodGet) + w.WriteHeader(http.StatusOK) + vs := values{ + "versions": "", "delimiter": "/", } testFormValues(t, r, vs) @@ -203,51 +203,51 @@ func TestBucketService_GetObjectVersions(t *testing.T) { `) - }) + }) - want := &BucketGetObjectVersionsResult { - XMLName: xml.Name { Local: "ListVersionsResult" }, - Name: "examplebucket-1250000000", - MaxKeys: 1000, - IsTruncated: false, - Delimiter: "/", - CommonPrefixes: []string { - "example-folder-1/", - "example-folder-2/", - }, - Version: []ListVersionsResultVersion { - { - Key: "example-object-1.jpg", - VersionId: "MTg0NDUxNzgxMjEzNTU3NTk1Mjg", - IsLatest: true, - LastModified: "2019-08-16T10:45:53.000Z", - ETag: "\"5d1143df07a17b23320d0da161e2819e\"", - Size: 30, - StorageClass: "STANDARD", - Owner: &Owner { - ID: "1250000000", - DisplayName: "1250000000", - }, - }, - }, - DeleteMarker: []ListVersionsResultDeleteMarker { - { - Key: "example-object-1.jpg", - VersionId: "MTg0NDUxNzgxMjEzNjE1OTcxMzM", - IsLatest: false, - LastModified: "2019-08-16T10:45:47.000Z", - Owner: &Owner { - ID: "1250000000", - DisplayName: "1250000000", - }, - }, - }, - } - opt := &BucketGetObjectVersionsOptions { - Delimiter: "/", - } - res, _, err := client.Bucket.GetObjectVersions(context.Background(), opt) - if err != nil { + want := &BucketGetObjectVersionsResult{ + XMLName: xml.Name{Local: "ListVersionsResult"}, + Name: "examplebucket-1250000000", + MaxKeys: 1000, + IsTruncated: false, + Delimiter: "/", + CommonPrefixes: []string{ + "example-folder-1/", + "example-folder-2/", + }, + Version: []ListVersionsResultVersion{ + { + Key: "example-object-1.jpg", + VersionId: "MTg0NDUxNzgxMjEzNTU3NTk1Mjg", + IsLatest: true, + LastModified: "2019-08-16T10:45:53.000Z", + ETag: "\"5d1143df07a17b23320d0da161e2819e\"", + Size: 30, + StorageClass: "STANDARD", + Owner: &Owner{ + ID: "1250000000", + DisplayName: "1250000000", + }, + }, + }, + DeleteMarker: []ListVersionsResultDeleteMarker{ + { + Key: "example-object-1.jpg", + VersionId: "MTg0NDUxNzgxMjEzNjE1OTcxMzM", + IsLatest: false, + LastModified: "2019-08-16T10:45:47.000Z", + Owner: &Owner{ + ID: "1250000000", + DisplayName: "1250000000", + }, + }, + }, + } + opt := &BucketGetObjectVersionsOptions{ + Delimiter: "/", + } + res, _, err := client.Bucket.GetObjectVersions(context.Background(), opt) + if err != nil { t.Fatalf("Bucket.GetObjectVersions returned error: %v", err) } if !reflect.DeepEqual(res, want) { diff --git a/ci.go b/ci.go index 60fcff0..ed1bc84 100644 --- a/ci.go +++ b/ci.go @@ -48,13 +48,14 @@ type PicImageInfo struct { Quality int `xml:"Quality,omitempty"` } type PicProcessObject struct { - Key string `xml:"Key,omitempty"` - Location string `xml:"Location,omitempty"` - Format string `xml:"Format,omitempty"` - Width int `xml:"Width,omitempty"` - Height int `xml:"Height,omitempty"` - Size int `xml:"Size,omitempty"` - Quality int `xml:"Quality,omitempty"` + Key string `xml:"Key,omitempty"` + Location string `xml:"Location,omitempty"` + Format string `xml:"Format,omitempty"` + Width int `xml:"Width,omitempty"` + Height int `xml:"Height,omitempty"` + Size int `xml:"Size,omitempty"` + Quality int `xml:"Quality,omitempty"` + WatermarkStatus int `xml:"WatermarkStatus,omitempty"` } type picOperationsHeader struct { @@ -171,7 +172,7 @@ type VideoAuditingJobDetail struct { CreationTime string `xml:",omitempty"` Object string `xml:",omitempty"` SnapshotCount string `xml:",omitempty"` - result int `xml:",omitempty"` + Result int `xml:",omitempty"` PornInfo *RecognitionInfo `xml:",omitempty"` TerrorismInfo *RecognitionInfo `xml:",omitempty"` PoliticsInfo *RecognitionInfo `xml:",omitempty"` diff --git a/ci_doc.go b/ci_doc.go new file mode 100644 index 0000000..b6a571e --- /dev/null +++ b/ci_doc.go @@ -0,0 +1,271 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +type DocProcessJobInput struct { + Object string `xml:"Object,omitempty"` +} + +type DocProcessJobOutput struct { + Region string `xml:"Region,omitempty"` + Bucket string `xml:"Bucket,omitempty"` + Object string `xml:"Object,omitempty"` +} + +type DocProcessJobDocProcess struct { + SrcType string `xml:"SrcType,omitempty"` + TgtType string `xml:"TgtType,omitempty"` + SheetId int `xml:"SheetId,omitempty"` + StartPage int `xml:"StartPage,omitempty"` + EndPage int `xml:"EndPage,omitempty"` + ImageParams string `xml:"ImageParams,omitempty"` + DocPassword string `xml:"DocPassword,omitempty"` + Comments int `xml:"Comments,omitempty"` + PaperDirection int `xml:"PaperDirection,omitempty"` + Quality int `xml:"Quality,omitempty"` + Zoom int `xml:"Zoom,omitempty"` +} + +type DocProcessJobDocProcessResult struct { + FailPageCount int `xml:",omitempty"` + SuccPageCount int `xml:"SuccPageCount,omitempty"` + TaskId string `xml:"TaskId,omitempty"` + TgtType string `xml:"TgtType,omitempty"` + TotalPageCount int `xml:"TotalPageCount,omitempty"` + PageInfo struct { + PageNo int `xml:"PageNo,omitempty"` + TgtUri string `xml:"TgtUri,omitempty"` + } `xml:"PageInfo,omitempty"` +} + +type DocProcessJobOperation struct { + Output *DocProcessJobOutput `xml:"Output,omitempty"` + DocProcess *DocProcessJobDocProcess `xml:"DocProcess,omitempty"` + DocProcessResult *DocProcessJobDocProcessResult `xml:"DocProcessResult,omitempty"` +} + +type DocProcessJobDetail struct { + Code string `xml:"Code,omitempty"` + Message string `xml:"Message,omitempty"` + JobId string `xml:"JobId,omitempty"` + Tag string `xml:"Tag,omitempty"` + State string `xml:"State,omitempty"` + CreationTime string `xml:"CreationTime,omitempty"` + QueueId string `xml:"QueueId,omitempty"` + Input *DocProcessJobInput `xml:"Input,omitempty"` + Operation *DocProcessJobOperation `xml:"Operation,omitempty"` +} + +type CreateDocProcessJobsOptions struct { + XMLName xml.Name `xml:"Request"` + Tag string `xml:"Tag,omitempty"` + Input *DocProcessJobInput `xml:"Input,omitempty"` + Operation *DocProcessJobOperation `xml:"Operation,omitempty"` + QueueId string `xml:"QueueId,omitempty"` +} + +type CreateDocProcessJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail DocProcessJobDetail `xml:"JobsDetail,omitempty"` +} + +func (s *CIService) CreateDocProcessJobs(ctx context.Context, opt *CreateDocProcessJobsOptions) (*CreateDocProcessJobsResult, *Response, error) { + var res CreateDocProcessJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs", + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessJobResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail *DocProcessJobDetail `xml:"JobsDetail,omitempty"` + NonExistJobIds string `xml:"NonExistJobIds,omitempty"` +} + +func (s *CIService) DescribeDocProcessJob(ctx context.Context, jobid string) (*DescribeDocProcessJobResult, *Response, error) { + var res DescribeDocProcessJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs/" + jobid, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessJobsOptions struct { + QueueId string `url:"queueId,omitempty"` + Tag string `url:"tag,omitempty"` + OrderByTime string `url:"orderByTime,omitempty"` + NextToken string `url:"nextToken,omitempty"` + Size int `url:"size,omitempty"` + States string `url:"states,omitempty"` + StartCreationTime string `url:"startCreationTime,omitempty"` + EndCreationTime string `url:"endCreationTime,omitempty"` +} + +type DescribeDocProcessJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail []DocProcessJobDetail `xml:"JobsDetail,omitempty"` + NextToken string `xml:"NextToken,omitempty"` +} + +func (s *CIService) DescribeDocProcessJobs(ctx context.Context, opt *DescribeDocProcessJobsOptions) (*DescribeDocProcessJobsResult, *Response, error) { + var res DescribeDocProcessJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessQueuesOptions struct { + QueueIds string `url:"queueIds,omitempty"` + State string `url:"state,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeDocProcessQueuesResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + QueueList []DocProcessQueue `xml:"QueueList,omitempty"` + NonExistPIDs []string `xml:"NonExistPIDs,omitempty"` +} + +type DocProcessQueue struct { + QueueId string `xml:"QueueId,omitempty"` + Name string `xml:"Name,omitempty"` + State string `xml:"State,omitempty"` + MaxSize int `xml:"MaxSize,omitempty"` + MaxConcurrent int `xml:"MaxConcurrent,omitempty"` + UpdateTime string `xml:"UpdateTime,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` + NotifyConfig *DocProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type DocProcessQueueNotifyConfig struct { + Url string `xml:"Url,omitempty"` + State string `xml:"State,omitempty"` + Type string `xml:"Type,omitempty"` + Event string `xml:"Event,omitempty"` +} + +func (s *CIService) DescribeDocProcessQueues(ctx context.Context, opt *DescribeDocProcessQueuesOptions) (*DescribeDocProcessQueuesResult, *Response, error) { + var res DescribeDocProcessQueuesResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docqueue", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type UpdateDocProcessQueueOptions struct { + XMLName xml.Name `xml:"Request"` + Name string `xml:"Name,omitempty"` + QueueID string `xml:"QueueID,omitempty"` + State string `xml:"State,omitempty"` + NotifyConfig *DocProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type UpdateDocProcessQueueResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId"` + Queue *DocProcessQueue `xml:"Queue"` +} + +func (s *CIService) UpdateDocProcessQueue(ctx context.Context, opt *UpdateDocProcessQueueOptions) (*UpdateDocProcessQueueResult, *Response, error) { + var res UpdateDocProcessQueueResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docqueue/" + opt.QueueID, + body: opt, + method: http.MethodPut, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessBucketsOptions struct { + Regions string `url:"regions,omitempty"` + BucketNames string `url:"bucketNames,omitempty"` + BucketName string `url:"bucketName,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeDocProcessBucketsResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + DocBucketList []DocProcessBucket `xml:"DocBucketList,omitempty"` +} +type DocProcessBucket struct { + BucketId string `xml:"BucketId,omitempty"` + Name string `xml:"Name,omitempty"` + Region string `xml:"Region,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` + AliasBucketId string `xml:"AliasBucketId,omitempty"` +} + +func (s *CIService) DescribeDocProcessBuckets(ctx context.Context, opt *DescribeDocProcessBucketsOptions) (*DescribeDocProcessBucketsResult, *Response, error) { + var res DescribeDocProcessBucketsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docbucket", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DocPreviewOptions struct { + SrcType string `url:"srcType,omitempty"` + Page int `url:"page,omitempty"` + ImageParams string `url:"ImageParams,omitempty"` + Sheet int `url:"sheet,omitempty"` + DstType string `url:"dstType,omitempty"` + Password string `url:"password,omitempty"` + Comment int `url:"comment,omitempty"` + ExcelPaperDirection int `url:"excelPaperDirection,omitempty"` + Quality int `url:"quality,omitempty"` + Zoom int `url:"zoom,omitempty"` +} + +func (s *CIService) DocPreview(ctx context.Context, name string, opt *DocPreviewOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?ci-process=doc-preview", + optQuery: opt, + method: http.MethodGet, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/cos.go b/cos.go index 1a25e2f..30ccd67 100644 --- a/cos.go +++ b/cos.go @@ -22,7 +22,7 @@ import ( const ( // Version current go sdk version - Version = "0.7.12" + Version = "0.7.13" userAgent = "cos-go-sdk-v5/" + Version contentTypeXML = "application/xml" defaultServiceBaseURL = "http://service.cos.myqcloud.com" diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 915c830..dcc85f9 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -226,6 +226,7 @@ func (s *CosTestSuite) TestVersionAndReplication() { } _, err := s.Client.Bucket.PutVersioning(context.Background(), opt) assert.Nil(s.T(), err, "PutVersioning Failed") + time.Sleep(time.Second) v, _, err := s.Client.Bucket.GetVersioning(context.Background()) assert.Nil(s.T(), err, "GetVersioning Failed") assert.Equal(s.T(), "Enabled", v.Status, "Get Wrong Version status") @@ -248,6 +249,7 @@ func (s *CosTestSuite) TestVersionAndReplication() { _, err = s.Client.Bucket.PutBucketReplication(context.Background(), repOpt) assert.Nil(s.T(), err, "PutBucketReplication Failed") + time.Sleep(time.Second) vr, _, err := s.Client.Bucket.GetBucketReplication(context.Background()) assert.Nil(s.T(), err, "GetBucketReplication Failed") for _, r := range vr.Rule { diff --git a/example/bucket/intelligenttiering.go b/example/bucket/intelligenttiering.go index 0256843..6bdae1a 100644 --- a/example/bucket/intelligenttiering.go +++ b/example/bucket/intelligenttiering.go @@ -48,13 +48,13 @@ func main() { }, }) - opt := &cos.BucketPutIntelligentTieringOptions { - Status: "Enabled", - Transition: &cos.BucketIntelligentTieringTransition { - Days: 30, - }, - } - _, err := c.Bucket.PutIntelligentTiering(context.Background(), opt) + opt := &cos.BucketPutIntelligentTieringOptions{ + Status: "Enabled", + Transition: &cos.BucketIntelligentTieringTransition{ + Days: 30, + }, + } + _, err := c.Bucket.PutIntelligentTiering(context.Background(), opt) log_status(err) res, _, err := c.Bucket.GetIntelligentTiering(context.Background()) log_status(err) diff --git a/example/object/ci_doc_process.go b/example/object/ci_doc_process.go new file mode 100644 index 0000000..c743d64 --- /dev/null +++ b/example/object/ci_doc_process.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func log_status(err error) { + if err == nil { + return + } + if cos.IsNotFoundError(err) { + // WARN + fmt.Println("WARN: Resource is not existed") + } else if e, ok := cos.IsCOSError(err); ok { + fmt.Printf("ERROR: Code: %v\n", e.Code) + fmt.Printf("ERROR: Message: %v\n", e.Message) + fmt.Printf("ERROR: Resource: %v\n", e.Resource) + fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) + // ERROR + } else { + fmt.Printf("ERROR: %v\n", err) + // ERROR + } +} + +func main() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + cu, _ := url.Parse("https://test-1259654469.ci.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u, CIURL: cu} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + // Notice when put a large file and set need the request body, might happend out of memory error. + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + // 1、UpdateDocProcessQueue + updateQueueOpt := &cos.UpdateDocProcessQueueOptions{ + Name: "queue-doc-process-1", + QueueID: "p111a8dd208104ce3b11c78398f658ca8", + State: "Active", + NotifyConfig: &cos.DocProcessQueueNotifyConfig{ + State: "Off", + }, + } + updateQueueRes, _, err := c.CI.UpdateDocProcessQueue(context.Background(), updateQueueOpt) + log_status(err) + fmt.Printf("%+v\n", updateQueueRes) + + // 2、DescribeDocProcessQueues + DescribeQueueOpt := &cos.DescribeDocProcessQueuesOptions{ + QueueIds: "p111a8dd208104ce3b11c78398f658ca8,p4318f85d2aa14c43b1dba6f9b78be9b3,aacb2bb066e9c4478834d4196e76c49d3", + PageNumber: 1, + PageSize: 2, + } + DescribeQueueRes, _, err := c.CI.DescribeDocProcessQueues(context.Background(), DescribeQueueOpt) + log_status(err) + fmt.Printf("%+v\n", DescribeQueueRes) + + // 3、DescribeDocProcessBuckets + BucketsOpt := &cos.DescribeDocProcessBucketsOptions{ + Regions: "All", + } + BucketsRes, _, err := c.CI.DescribeDocProcessBuckets(context.Background(), BucketsOpt) + log_status(err) + fmt.Printf("%+v\n", BucketsRes) + + // 4、CreateDocProcessJobs + createJobOpt := &cos.CreateDocProcessJobsOptions{ + Tag: "DocProcess", + Input: &cos.DocProcessJobInput{ + Object: "form.pdf", + }, + Operation: &cos.DocProcessJobOperation{ + Output: &cos.DocProcessJobOutput{ + Region: "ap-guangzhou", + Object: "test-doc${Number}", + Bucket: "test-1259654469", + }, + DocProcess: &cos.DocProcessJobDocProcess{ + TgtType: "png", + StartPage: 1, + EndPage: -1, + ImageParams: "watermark/1/image/aHR0cDovL3Rlc3QwMDUtMTI1MTcwNDcwOC5jb3MuYXAtY2hvbmdxaW5nLm15cWNsb3VkLmNvbS8xLmpwZw==/gravity/southeast", + }, + }, + QueueId: "p111a8dd208104ce3b11c78398f658ca8", + } + createJobRes, _, err := c.CI.CreateDocProcessJobs(context.Background(), createJobOpt) + log_status(err) + fmt.Printf("%+v\n", createJobRes.JobsDetail) + + // 5、DescribeDocProcessJob + DescribeJobRes, _, err := c.CI.DescribeDocProcessJob(context.Background(), createJobRes.JobsDetail.JobId) + log_status(err) + fmt.Printf("%+v\n", DescribeJobRes.JobsDetail) + + // 6、DescribeDocProcessJobs + DescribeJobsOpt := &cos.DescribeDocProcessJobsOptions{ + QueueId: "p111a8dd208104ce3b11c78398f658ca8", + Tag: "DocProcess", + } + DescribeJobsRes, _, err := c.CI.DescribeDocProcessJobs(context.Background(), DescribeJobsOpt) + log_status(err) + fmt.Printf("%+v\n", DescribeJobsRes) + + // 7、doc-preview + opt := &cos.DocPreviewOptions{ + Page: 1, + } + resp, err := c.CI.DocPreview(context.Background(), "form.pdf", opt) + log_status(err) + fd, _ := os.OpenFile("form.pdf", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + io.Copy(fd, resp.Body) + fd.Close() + +} diff --git a/example/object/get.go b/example/object/get.go index 2650030..ff44ebf 100644 --- a/example/object/get.go +++ b/example/object/get.go @@ -19,7 +19,7 @@ func log_status(err error) { } if cos.IsNotFoundError(err) { // WARN - fmt.Println("WARN: Resource is not existed") + fmt.Println("WARN: Resource is not existed") } else if e, ok := cos.IsCOSError(err); ok { fmt.Printf("ERROR: Code: %v\n", e.Code) fmt.Printf("ERROR: Message: %v\n", e.Message) @@ -33,7 +33,7 @@ func log_status(err error) { } func main() { - u, _ := url.Parse("https://test-1253846586.cos.ap-guangzhou.myqcloud.com") + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") b := &cos.BaseURL{BucketURL: u} c := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ @@ -48,8 +48,8 @@ func main() { }, }) - // Case1 Download object into ReadCloser(). the body needs to be closed - name := "test/hello.txt" + // Case1 通过resp.Body下载对象，Body需要关闭 + name := "test/example" resp, err := c.Object.Get(context.Background(), name, nil) log_status(err) @@ -57,8 +57,8 @@ func main() { resp.Body.Close() fmt.Printf("%s\n", string(bs)) - // Case2 Download object to local file. the body needs to be closed - fd, err := os.OpenFile("hello.txt", os.O_WRONLY|os.O_CREATE, 0660) + // Case2 下载对象到文件. Body需要关闭 + fd, err := os.OpenFile("test", os.O_WRONLY|os.O_CREATE, 0660) log_status(err) defer fd.Close() @@ -68,11 +68,11 @@ func main() { io.Copy(fd, resp.Body) resp.Body.Close() - // Case3 Download object to local file path - _, err = c.Object.GetToFile(context.Background(), name, "hello_1.txt", nil) + // Case3 下载对象到文件 + _, err = c.Object.GetToFile(context.Background(), name, "test", nil) log_status(err) - // Case4 Download object with range header, can used to concurrent download + // Case4 range下载对象，可以根据range实现并发下载 opt := &cos.ObjectGetOptions{ ResponseContentType: "text/html", Range: "bytes=0-3", @@ -82,4 +82,11 @@ func main() { bs, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() fmt.Printf("%s\n", string(bs)) + + // Case5 下载对象到文件，查看下载进度 + opt = &cos.ObjectGetOptions{ + Listener: &cos.DefaultProgressListener{}, + } + _, err = c.Object.GetToFile(context.Background(), name, "test", opt) + log_status(err) } diff --git a/example/object/put.go b/example/object/put.go index a9f97f3..0f4b12b 100644 --- a/example/object/put.go +++ b/example/object/put.go @@ -19,7 +19,7 @@ func log_status(err error) { } if cos.IsNotFoundError(err) { // WARN - fmt.Println("WARN: Resource is not existed") + fmt.Println("WARN: Resource is not existed") } else if e, ok := cos.IsCOSError(err); ok { fmt.Printf("ERROR: Code: %v\n", e.Code) fmt.Printf("ERROR: Message: %v\n", e.Message) @@ -44,20 +44,19 @@ func main() { // Notice when put a large file and set need the request body, might happend out of memory error. RequestBody: false, ResponseHeader: true, - ResponseBody: true, + ResponseBody: false, }, }, }) - // Case1 normal put object - name := "test/objectPut.go" + // Case1 上传对象 + name := "test/example" f := strings.NewReader("test") _, err := c.Object.Put(context.Background(), name, f, nil) log_status(err) - // Case2 put object with the options - name = "test/put_option.go" + // Case2 使用options上传对象 f = strings.NewReader("test xxx") opt := &cos.ObjectPutOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ @@ -71,7 +70,11 @@ func main() { _, err = c.Object.Put(context.Background(), name, f, opt) log_status(err) - // Case3 put object by local file path + // Case3 通过本地文件上传对象 _, err = c.Object.PutFromFile(context.Background(), name, "./test", nil) log_status(err) + + // Case4 查看上传进度 + opt.ObjectPutHeaderOptions.Listener = &cos.DefaultProgressListener{} + _, err = c.Object.PutFromFile(context.Background(), name, "./test", opt) } diff --git a/example/object/put_with_timeout.go b/example/object/put_with_timeout.go new file mode 100644 index 0000000..e86b8a4 --- /dev/null +++ b/example/object/put_with_timeout.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "time" + + "net/http" + + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func log_status(err error) { + if err == nil { + return + } + if cos.IsNotFoundError(err) { + // WARN + fmt.Println("WARN: Resource is not existed") + } else if e, ok := cos.IsCOSError(err); ok { + fmt.Printf("ERROR: Code: %v\n", e.Code) + fmt.Printf("ERROR: Message: %v\n", e.Message) + fmt.Printf("ERROR: Resource: %v\n", e.Resource) + fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) + // ERROR + } else { + fmt.Printf("ERROR: %v\n", err) + // ERROR + } +} + +func main() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + // Notice when put a large file and set need the request body, might happend out of memory error. + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + Timeout: 5 * time.Second, // HTTP超时时间 + }) + + // Case1 上传对象 + name := "test/example" + // Case3 通过本地文件上传对象 + ctx, _ := context.WithTimeout(context.Background(), 2*time.Second) // context超时时间 + _, err := c.Object.PutFromFile(ctx, name, "./test", nil) // 请求的超时时间为 min{context超时时间， HTTP超时时间} + log_status(err) +} diff --git a/example/object/upload.go b/example/object/upload.go index 7894fda..54a6f7d 100644 --- a/example/object/upload.go +++ b/example/object/upload.go @@ -49,9 +49,26 @@ func main() { }, }) + // Case1 多线程上传对象 + opt := &cos.MultiUploadOptions{ + ThreadPoolSize: 3, + } v, _, err := c.Object.Upload( - context.Background(), "gomulput1G", "./test1G", nil, + context.Background(), "gomulput1G", "./test1G", opt, + ) + log_status(err) + fmt.Printf("Case1 done, %v\n", v) + + // Case2 多线程上传对象，查看上传进度 + opt.OptIni = &cos.InitiateMultipartUploadOptions{ + nil, + &cos.ObjectPutHeaderOptions{ + Listener: &cos.DefaultProgressListener{}, + }, + } + v, _, err = c.Object.Upload( + context.Background(), "gomulput1G", "./test1G", opt, ) log_status(err) - fmt.Println(v) + fmt.Printf("Case2 done, %v\n", v) } diff --git a/example/object/uploadPart.go b/example/object/uploadPart.go index 945bfe5..0a3c5fa 100644 --- a/example/object/uploadPart.go +++ b/example/object/uploadPart.go @@ -41,7 +41,7 @@ func initUpload(c *cos.Client, name string) *cos.InitiateMultipartUploadResult { } func main() { - u, _ := url.Parse("https://test-1253846586.cos.ap-guangzhou.myqcloud.com") + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") b := &cos.BaseURL{BucketURL: u} c := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ @@ -49,20 +49,50 @@ func main() { SecretKey: os.Getenv("COS_SECRETKEY"), Transport: &debug.DebugRequestTransport{ RequestHeader: true, - RequestBody: true, + RequestBody: false, ResponseHeader: true, - ResponseBody: true, + ResponseBody: false, }, }, }) + optcom := &cos.CompleteMultipartUploadOptions{} name := "test/test_multi_upload.go" up := initUpload(c, name) uploadID := up.UploadID + fd, err := os.Open("test") + if err != nil { + fmt.Printf("Open File Error: %v\n", err) + return + } + defer fd.Close() + stat, err := fd.Stat() + if err != nil { + fmt.Printf("Stat File Error: %v\n", err) + return + } + opt := &cos.ObjectUploadPartOptions{ + Listener: &cos.DefaultProgressListener{}, + ContentLength: int(stat.Size()), + } + resp, err := c.Object.UploadPart( + context.Background(), name, uploadID, 1, fd, opt, + ) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: 1, ETag: resp.Header.Get("ETag"), + }) + log_status(err) + f := strings.NewReader("test heoo") - _, err := c.Object.UploadPart( - context.Background(), name, uploadID, 1, f, nil, + resp, err = c.Object.UploadPart( + context.Background(), name, uploadID, 2, f, nil, ) log_status(err) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: 2, ETag: resp.Header.Get("ETag"), + }) + + _, _, err = c.Object.CompleteMultipartUpload(context.Background(), name, uploadID, optcom) + log_status(err) } diff --git a/helper.go b/helper.go index 8e98348..52fdeee 100644 --- a/helper.go +++ b/helper.go @@ -5,8 +5,11 @@ import ( "crypto/md5" "crypto/sha1" "fmt" + "io" "net/http" "net/url" + "os" + "strings" ) // 计算 md5 或 sha1 时的分块大小 @@ -112,3 +115,28 @@ func DecodeURIComponent(s string) (string, error) { func EncodeURIComponent(s string) string { return encodeURIComponent(s) } + +func GetReaderLen(reader io.Reader) (length int64, err error) { + switch v := reader.(type) { + case *bytes.Buffer: + length = int64(v.Len()) + case *bytes.Reader: + length = int64(v.Len()) + case *strings.Reader: + length = int64(v.Len()) + case *os.File: + stat, ferr := v.Stat() + if ferr != nil { + err = fmt.Errorf("can't get reader length: %s", ferr.Error()) + } else { + length = stat.Size() + } + case *io.LimitedReader: + length = int64(v.N) + case FixedLengthReader: + length = v.Size() + default: + err = fmt.Errorf("can't get reader content length, unkown reader type") + } + return +} diff --git a/object.go b/object.go index 8933c69..5d3337a 100644 --- a/object.go +++ b/object.go @@ -12,6 +12,7 @@ import ( "net/url" "os" "sort" + "strconv" "strings" "time" ) @@ -35,6 +36,9 @@ type ObjectGetOptions struct { XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + // 下载进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // presignedURLTestingOptions is the opt of presigned url @@ -65,6 +69,14 @@ func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOpti disableCloseBody: true, } resp, err := s.client.send(ctx, &sendOpt) + + if opt != nil && opt.Listener != nil { + if err == nil && resp != nil { + if totalBytes, e := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64); e == nil { + resp.Body = TeeReader(resp.Body, nil, totalBytes, opt.Listener) + } + } + } return resp, err } @@ -152,6 +164,9 @@ type ObjectPutHeaderOptions struct { //兼容其他自定义头部 XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + // 上传进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // ObjectPutOptions the options of put object @@ -166,6 +181,14 @@ type ObjectPutOptions struct { // // https://www.qcloud.com/document/product/436/7749 func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) { + if opt != nil && opt.Listener != nil { + totalBytes, err := GetReaderLen(r) + if err != nil { + return nil, err + } + r = TeeReader(r, nil, totalBytes, opt.Listener) + } + sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: "/" + encodeURIComponent(name), @@ -174,6 +197,7 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt * optHeader: opt, } resp, err := s.client.send(ctx, &sendOpt) + return resp, err } @@ -571,27 +595,27 @@ func DividePart(fileSize int64) (int64, int64) { return partNum, partSize } -func SplitFileIntoChunks(filePath string, partSize int64) ([]Chunk, int, error) { +func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, error) { if filePath == "" { - return nil, 0, errors.New("filePath invalid") + return 0, nil, 0, errors.New("filePath invalid") } file, err := os.Open(filePath) if err != nil { - return nil, 0, err + return 0, nil, 0, err } defer file.Close() stat, err := file.Stat() if err != nil { - return nil, 0, err + return 0, nil, 0, err } var partNum int64 if partSize > 0 { partSize = partSize * 1024 * 1024 partNum = stat.Size() / partSize if partNum >= 10000 { - return nil, 0, errors.New("Too many parts, out of 10000") + return 0, nil, 0, errors.New("Too many parts, out of 10000") } } else { partNum, partSize = DividePart(stat.Size()) @@ -614,7 +638,7 @@ func SplitFileIntoChunks(filePath string, partSize int64) ([]Chunk, int, error) partNum++ } - return chunks, int(partNum), nil + return int64(stat.Size()), chunks, int(partNum), nil } @@ -707,7 +731,7 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string opt = &MultiUploadOptions{} } // 1.Get the file chunk - chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize) + totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize) if err != nil { return nil, nil, err } @@ -768,6 +792,15 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string go worker(s, chjobs, chresults) } + // progress started event + var listener ProgressListener + var consumedBytes int64 + if opt.OptIni != nil { + listener = opt.OptIni.Listener + } + event := newProgressEvent(ProgressStartedEvent, 0, 0, totalBytes) + progressCallback(listener, event) + // 4.Push jobs for _, chunk := range chunks { if chunk.Done { @@ -798,22 +831,34 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string optcom.Parts = append(optcom.Parts, Object{ PartNumber: chunks[i].Number, ETag: chunks[i].ETag}, ) + consumedBytes += chunks[i].Size + event = newProgressEvent(ProgressDataEvent, chunks[i].Size, consumedBytes, totalBytes) + progressCallback(listener, event) continue } res := <-chresults // Notice one part fail can not get the etag according. if res.Resp == nil || res.err != nil { // Some part already fail, can not to get the header inside. - return nil, nil, fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error()) + err := fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error()) + event = newProgressEvent(ProgressFailedEvent, 0, consumedBytes, totalBytes, err) + progressCallback(listener, event) + return nil, nil, err } // Notice one part fail can not get the etag according. etag := res.Resp.Header.Get("ETag") optcom.Parts = append(optcom.Parts, Object{ PartNumber: res.PartNumber, ETag: etag}, ) + consumedBytes += chunks[res.PartNumber-1].Size + event = newProgressEvent(ProgressDataEvent, chunks[res.PartNumber-1].Size, consumedBytes, totalBytes) + progressCallback(listener, event) } sort.Sort(ObjectList(optcom.Parts)) + event = newProgressEvent(ProgressCompletedEvent, 0, consumedBytes, totalBytes) + progressCallback(listener, event) + v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom) return v, resp, err diff --git a/object_part.go b/object_part.go index ee181b8..718899c 100644 --- a/object_part.go +++ b/object_part.go @@ -50,6 +50,9 @@ type ObjectUploadPartOptions struct { XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + // 上传进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // UploadPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。 @@ -61,6 +64,13 @@ type ObjectUploadPartOptions struct { // // https://www.qcloud.com/document/product/436/7750 func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) { + if opt != nil && opt.Listener != nil { + totalBytes, err := GetReaderLen(r) + if err != nil { + return nil, err + } + r = TeeReader(r, nil, totalBytes, opt.Listener) + } u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, diff --git a/object_test.go b/object_test.go index fa63e42..cce6677 100644 --- a/object_test.go +++ b/object_test.go @@ -129,7 +129,7 @@ func TestObjectService_Options(t *testing.T) { }) opt := &ObjectOptionsOptions{ - Origin: "www.qq.com", + Origin: "www.qq.com", AccessControlRequestMethod: "PUT", } diff --git a/progress.go b/progress.go new file mode 100644 index 0000000..65d3185 --- /dev/null +++ b/progress.go @@ -0,0 +1,135 @@ +package cos + +import ( + "fmt" + "io" +) + +type ProgressEventType int + +const ( + // 数据开始传输 + ProgressStartedEvent ProgressEventType = iota + // 数据传输中 + ProgressDataEvent + // 数据传输完成, 但不能表示对应API调用完成 + ProgressCompletedEvent + // 只有在数据传输时发生错误才会返回 + ProgressFailedEvent +) + +type ProgressEvent struct { + EventType ProgressEventType + RWBytes int64 + ConsumedBytes int64 + TotalBytes int64 + Err error +} + +func newProgressEvent(eventType ProgressEventType, rwBytes, consumed, total int64, err ...error) *ProgressEvent { + event := &ProgressEvent{ + EventType: eventType, + RWBytes: rwBytes, + ConsumedBytes: consumed, + TotalBytes: total, + } + if len(err) > 0 { + event.Err = err[0] + } + return event +} + +// 用户自定义Listener需要实现该方法 +type ProgressListener interface { + ProgressChangedCallback(event *ProgressEvent) +} + +func progressCallback(listener ProgressListener, event *ProgressEvent) { + if listener != nil && event != nil { + listener.ProgressChangedCallback(event) + } +} + +type teeReader struct { + reader io.Reader + writer io.Writer + consumedBytes int64 + totalBytes int64 + listener ProgressListener +} + +func (r *teeReader) Read(p []byte) (int, error) { + if r.consumedBytes == 0 { + event := newProgressEvent(ProgressStartedEvent, 0, r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + + n, err := r.reader.Read(p) + if err != nil && err != io.EOF { + event := newProgressEvent(ProgressFailedEvent, 0, r.consumedBytes, r.totalBytes, err) + progressCallback(r.listener, event) + } + if n > 0 { + r.consumedBytes += int64(n) + if r.writer != nil { + if n, err := r.writer.Write(p[:n]); err != nil { + return n, err + } + } + if r.listener != nil { + event := newProgressEvent(ProgressDataEvent, int64(n), r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + } + + if err == io.EOF { + event := newProgressEvent(ProgressCompletedEvent, int64(n), r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + + return n, err +} + +func (r *teeReader) Close() error { + if rc, ok := r.reader.(io.ReadCloser); ok { + return rc.Close() + } + return nil +} + +func TeeReader(reader io.Reader, writer io.Writer, total int64, listener ProgressListener) *teeReader { + return &teeReader{ + reader: reader, + writer: writer, + consumedBytes: 0, + totalBytes: total, + listener: listener, + } +} + +type FixedLengthReader interface { + io.Reader + Size() int64 +} + +type DefaultProgressListener struct { +} + +func (l *DefaultProgressListener) ProgressChangedCallback(event *ProgressEvent) { + switch event.EventType { + case ProgressStartedEvent: + fmt.Printf("Transfer Start [ConsumedBytes/TotalBytes: %d/%d]\n", + event.ConsumedBytes, event.TotalBytes) + case ProgressDataEvent: + fmt.Printf("\rTransfer Data [ConsumedBytes/TotalBytes: %d/%d, %d%%]", + event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes) + case ProgressCompletedEvent: + fmt.Printf("\nTransfer Complete [ConsumedBytes/TotalBytes: %d/%d]\n", + event.ConsumedBytes, event.TotalBytes) + case ProgressFailedEvent: + fmt.Printf("\nTransfer Failed [ConsumedBytes/TotalBytes: %d/%d] [Err: %v]\n", + event.ConsumedBytes, event.TotalBytes, event.Err) + default: + fmt.Printf("Progress Changed Error: unknown progress event type\n") + } +} diff --git a/service_test.go b/service_test.go index f45a57c..b5c1cda 100644 --- a/service_test.go +++ b/service_test.go @@ -48,13 +48,13 @@ func TestServiceService_Get(t *testing.T) { }, Buckets: []Bucket{ { - Name: "huadong-1253846586", - Region: "ap-shanghai", + Name: "huadong-1253846586", + Region: "ap-shanghai", CreationDate: "2017-06-16T13:08:28Z", }, { - Name: "huanan-1253846586", - Region: "ap-guangzhou", + Name: "huanan-1253846586", + Region: "ap-guangzhou", CreationDate: "2017-06-10T09:00:07Z", }, },