diff --git a/cos.go b/cos.go index 7969c05..72df0bf 100644 --- a/cos.go +++ b/cos.go @@ -22,7 +22,7 @@ import ( const ( // Version current go sdk version - Version = "0.7.23" + Version = "0.7.24" userAgent = "cos-go-sdk-v5/" + Version contentTypeXML = "application/xml" defaultServiceBaseURL = "http://service.cos.myqcloud.com" @@ -71,7 +71,8 @@ func NewBucketURL(bucketName, region string, secure bool) *url.URL { } type Config struct { - EnableCRC bool + EnableCRC bool + RequestBodyClose bool } // Client is a client manages communication with the COS API. @@ -119,7 +120,8 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { UserAgent: userAgent, BaseURL: baseURL, Conf: &Config{ - EnableCRC: true, + EnableCRC: true, + RequestBodyClose: false, }, } c.common.client = c @@ -182,6 +184,9 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s if c.Host != "" { req.Host = c.Host } + if c.Conf.RequestBodyClose { + req.Close = true + } return } diff --git a/helper.go b/helper.go index 3541405..7b4acbf 100644 --- a/helper.go +++ b/helper.go @@ -16,6 +16,7 @@ import ( // 单次上传文件最大为5GB const singleUploadMaxLength = 5 * 1024 * 1024 * 1024 +const singleUploadThreshold = 32 * 1024 * 1024 // 计算 md5 或 sha1 时的分块大小 const calDigestBlockSize = 1024 * 1024 * 10 diff --git a/object.go b/object.go index 149d33a..bf9ef66 100644 --- a/object.go +++ b/object.go @@ -217,14 +217,24 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt // PutFromFile put object from local file // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory -func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) { - fd, err := os.Open(filePath) - if err != nil { - return nil, err +func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) { + nr := 0 + for nr < 3 { + fd, e := os.Open(filePath) + if e != nil { + err = e + return + } + resp, err = s.Put(ctx, name, fd, opt) + if err != nil { + nr++ + fd.Close() + continue + } + fd.Close() + break } - defer fd.Close() - - return s.Put(ctx, name, fd, opt) + return } // ObjectCopyHeaderOptions is the head option of the Copy @@ -778,9 +788,12 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string } defer fd.Close() localcrc, err = calCRC64(fd) + if err != nil { + return nil, nil, err + } } // filesize=0 , use simple upload - if partNum == 0 { + if partNum == 0 || partNum == 1 { var opt0 *ObjectPutOptions if opt.OptIni != nil { opt0 = &ObjectPutOptions{ diff --git a/object_test.go b/object_test.go index 5eee886..fcd8f2e 100644 --- a/object_test.go +++ b/object_test.go @@ -375,13 +375,13 @@ func TestObjectService_Upload(t *testing.T) { } defer os.Remove(filePath) // 源文件内容 - b := make([]byte, 1024*1024*10) + b := make([]byte, 1024*1024*33) _, err = rand.Read(b) newfile.Write(b) newfile.Close() // 已上传内容, 10个分块 - rb := make([][]byte, 10) + rb := make([][]byte, 33) uploadid := "test-cos-multiupload-uploadid" partmap := make(map[int64]int) mux.HandleFunc("/test.go.upload", func(w http.ResponseWriter, r *http.Request) { @@ -423,9 +423,7 @@ func TestObjectService_Upload(t *testing.T) { // 完成分块上传 tb := crc64.MakeTable(crc64.ECMA) crc := uint64(0) - ccv := make([]uint64, 10) - for i, v := range rb { - ccv[i] = crc64.Update(0, crc64.MakeTable(crc64.ECMA), v) + for _, v := range rb { crc = crc64.Update(crc, tb, v) } w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10)) @@ -450,3 +448,69 @@ func TestObjectService_Upload(t *testing.T) { t.Fatalf("Object.Upload returned error: %v", err) } } + +func TestObjectService_Upload2(t *testing.T) { + setup() + defer teardown() + + filePath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filePath) + if err != nil { + t.Fatalf("create tmp file failed") + } + defer os.Remove(filePath) + // 源文件内容 + b := make([]byte, 1024*1024*3) + _, err = rand.Read(b) + newfile.Write(b) + newfile.Close() + + tb := crc64.MakeTable(crc64.ECMA) + realcrc := crc64.Update(0, tb, b) + name := "test/hello.txt" + retry := 0 + final := 4 + mux.HandleFunc("/test/hello.txt", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, http.MethodPut) + testHeader(t, r, "x-cos-acl", "private") + testHeader(t, r, "Content-Type", "text/html") + + if retry%2 == 0 { + bs, _ := ioutil.ReadAll(r.Body) + crc := crc64.Update(0, tb, bs) + if !reflect.DeepEqual(bs, b) { + t.Errorf("Object.Put request body Error") + } + if !reflect.DeepEqual(crc, realcrc) { + t.Errorf("Object.Put crc: %v, want: %v", crc, realcrc) + } + w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10)) + if retry != final { + w.WriteHeader(http.StatusGatewayTimeout) + } + } else { + w.Header().Add("x-cos-hash-crc64ecma", "123456789") + } + }) + + mopt := &MultiUploadOptions{ + OptIni: &InitiateMultipartUploadOptions{ + ObjectPutHeaderOptions: &ObjectPutHeaderOptions{ + ContentType: "text/html", + }, + ACLHeaderOptions: &ACLHeaderOptions{ + XCosACL: "private", + }, + }, + } + for retry <= final { + _, _, err := client.Object.Upload(context.Background(), name, filePath, mopt) + if retry < final && err == nil { + t.Fatalf("Error must not nil when retry < final") + } + if retry == final && err != nil { + t.Fatalf("Put Error: %v", err) + } + retry++ + } +}