diff --git a/CHANGELOG.md b/CHANGELOG.md index a5368d1..c004302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # ChangeLog - Aliyun OSS SDK for Go +## 版本号：v2.2.9 日期：2023-08-25 +### 变更内容 +- 增加：support force path style option. +- 增加：support context.Context option. +- 修改：remove LifecycleFilterNot.Prefix omitempty attribute. + + +## 版本号：v2.2.8 日期：2023-07-31 +### 变更内容 +- 增加：support EnvironmentVariableCredentialsProvider +- 增加：support describe regions api. +- 增加：support create bucket with server encryption parameters. +- 增加：support referer black list. +- 增加：support aysnc process object api. +- 增加：support ObjectSizeGreaterThan and ObjectSizeLessThan in lifecycle rule. +- 增加：add DeepColdArchive storage class. +- 修复：fix bug. + ## 版本号：v2.2.7 日期：2023-03-23 ### 变更内容 - 增加：support get info form EC & x-oss-err. diff --git a/README-CN.md b/README-CN.md index 9f1a3d3..df65354 100644 --- a/README-CN.md +++ b/README-CN.md @@ -13,7 +13,7 @@ > - 使用此SDK，用户可以方便地在任何应用、任何时间、任何地点上传，下载和管理数据。 ## 版本 -> - Current version: v2.2.7 +> - Current version: v2.2.9 ## 运行环境 > - Go 1.5及以上。 diff --git a/README.md b/README.md index e3f6e84..0ef5f18 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ > - With this SDK, you can upload, download and manage data on any app anytime and anywhere conveniently. ## Version -> - Current version: v2.2.7 +> - Current version: v2.2.9 ## Running Environment > - Go 1.5 or above. ## Installing ### Install the SDK through GitHub -> - Run the 'go get github.com/aliyun/aliyun-oss-go-sdk/oss' command to get the remote code package. +> - Run the 'go get github.com/aliyun/aliyun-oss-go-sdk' command to get the remote code package. > - Use 'import "github.com/aliyun/aliyun-oss-go-sdk/oss"' in your code to introduce OSS Go SDK package. ## Getting Started diff --git a/oss/bucket.go b/oss/bucket.go index 1a7df0b..28c27c5 100644 --- a/oss/bucket.go +++ b/oss/bucket.go @@ -2,6 +2,7 @@ package oss import ( "bytes" + "context" "crypto/md5" "encoding/base64" "encoding/xml" @@ -29,11 +30,11 @@ type Bucket struct { // objectKey the object key in UTF-8 encoding. The length must be between 1 and 1023, and cannot start with "/" or "\". // reader io.Reader instance for reading the data for uploading // options the options for uploading the object. The valid options here are CacheControl, ContentDisposition, ContentEncoding -// Expires, ServerSideEncryption, ObjectACL and Meta. Refer to the link below for more details. -// https://www.alibabacloud.com/help/en/object-storage-service/latest/putobject // -// error it's nil if no error, otherwise it's an error object. +// Expires, ServerSideEncryption, ObjectACL and Meta. Refer to the link below for more details. +// https://www.alibabacloud.com/help/en/object-storage-service/latest/putobject // +// error it's nil if no error, otherwise it's an error object. func (bucket Bucket) PutObject(objectKey string, reader io.Reader, options ...Option) error { opts := AddContentType(options, objectKey) @@ -57,7 +58,6 @@ func (bucket Bucket) PutObject(objectKey string, reader io.Reader, options ...Op // options the options for uploading the object. Refer to the parameter options in PutObject for more details. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) PutObjectFromFile(objectKey, filePath string, options ...Option) error { fd, err := os.Open(filePath) if err != nil { @@ -87,7 +87,6 @@ func (bucket Bucket) PutObjectFromFile(objectKey, filePath string, options ...Op // // Response the response from OSS. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DoPutObject(request *PutObjectRequest, options []Option) (*Response, error) { isOptSet, _, _ := IsOptionSet(options, HTTPHeaderContentType) if !isOptSet { @@ -118,12 +117,12 @@ func (bucket Bucket) DoPutObject(request *PutObjectRequest, options []Option) (* // // objectKey the object key. // options the options for downloading the object. The valid values are: Range, IfModifiedSince, IfUnmodifiedSince, IfMatch, -// IfNoneMatch, AcceptEncoding. For more details, please check out: -// https://www.alibabacloud.com/help/en/object-storage-service/latest/getobject +// +// IfNoneMatch, AcceptEncoding. For more details, please check out: +// https://www.alibabacloud.com/help/en/object-storage-service/latest/getobject // // io.ReadCloser reader instance for reading data from response. It must be called close() after the usage and only valid when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObject(objectKey string, options ...Option) (io.ReadCloser, error) { result, err := bucket.DoGetObject(&GetObjectRequest{objectKey}, options) if err != nil { @@ -140,7 +139,6 @@ func (bucket Bucket) GetObject(objectKey string, options ...Option) (io.ReadClos // options the options for downloading the object. Refer to the parameter options in method GetObject for more details. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectToFile(objectKey, filePath string, options ...Option) error { tempFilePath := filePath + TempFileSuffix @@ -190,7 +188,6 @@ func (bucket Bucket) GetObjectToFile(objectKey, filePath string, options ...Opti // // GetObjectResult the result instance of getting the object. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DoGetObject(request *GetObjectRequest, options []Option) (*GetObjectResult, error) { params, _ := GetRawParams(options) resp, err := bucket.do("GET", request.ObjectKey, params, options, nil, nil) @@ -225,13 +222,13 @@ func (bucket Bucket) DoGetObject(request *GetObjectRequest, options []Option) (* // srcObjectKey the source object to copy. // destObjectKey the target object to copy. // options options for copying an object. You can specify the conditions of copy. The valid conditions are CopySourceIfMatch, -// CopySourceIfNoneMatch, CopySourceIfModifiedSince, CopySourceIfUnmodifiedSince, MetadataDirective. -// Also you can specify the target object's attributes, such as CacheControl, ContentDisposition, ContentEncoding, Expires, -// ServerSideEncryption, ObjectACL, Meta. Refer to the link below for more details : -// https://www.alibabacloud.com/help/en/object-storage-service/latest/copyobject // -// error it's nil if no error, otherwise it's an error object. +// CopySourceIfNoneMatch, CopySourceIfModifiedSince, CopySourceIfUnmodifiedSince, MetadataDirective. +// Also you can specify the target object's attributes, such as CacheControl, ContentDisposition, ContentEncoding, Expires, +// ServerSideEncryption, ObjectACL, Meta. Refer to the link below for more details : +// https://www.alibabacloud.com/help/en/object-storage-service/latest/copyobject // +// error it's nil if no error, otherwise it's an error object. func (bucket Bucket) CopyObject(srcObjectKey, destObjectKey string, options ...Option) (CopyObjectResult, error) { var out CopyObjectResult @@ -264,12 +261,10 @@ func (bucket Bucket) CopyObject(srcObjectKey, destObjectKey string, options ...O // options copy options, check out parameter options in function CopyObject for more details. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) CopyObjectTo(destBucketName, destObjectKey, srcObjectKey string, options ...Option) (CopyObjectResult, error) { return bucket.copy(srcObjectKey, destBucketName, destObjectKey, options...) } -// // CopyObjectFrom copies the object to another bucket. // // srcBucketName source bucket name. @@ -278,7 +273,6 @@ func (bucket Bucket) CopyObjectTo(destBucketName, destObjectKey, srcObjectKey st // options copy options. Check out parameter options in function CopyObject. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) CopyObjectFrom(srcBucketName, srcObjectKey, destObjectKey string, options ...Option) (CopyObjectResult, error) { destBucketName := bucket.BucketName var out CopyObjectResult @@ -309,7 +303,11 @@ func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, op return out, err } params := map[string]interface{}{} - resp, err := bucket.Client.Conn.Do("PUT", destBucketName, destObjectKey, params, headers, nil, 0, nil) + + ctxArg, _ := FindOption(options, contextArg, nil) + ctx, _ := ctxArg.(context.Context) + + resp, err := bucket.Client.Conn.DoWithContext(ctx, "PUT", destBucketName, destObjectKey, params, headers, nil, 0, nil) // get response header respHeader, _ := FindOption(options, responseHeader, nil) @@ -340,11 +338,11 @@ func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, op // reader io.Reader. The read instance for reading the data to append. // appendPosition the start position to append. // destObjectProperties the options for the first appending, such as CacheControl, ContentDisposition, ContentEncoding, -// Expires, ServerSideEncryption, ObjectACL. +// +// Expires, ServerSideEncryption, ObjectACL. // // int64 the next append position, it's valid when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) AppendObject(objectKey string, reader io.Reader, appendPosition int64, options ...Option) (int64, error) { request := &AppendObjectRequest{ ObjectKey: objectKey, @@ -367,7 +365,6 @@ func (bucket Bucket) AppendObject(objectKey string, reader io.Reader, appendPosi // // AppendObjectResult the result object for appending object. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Option) (*AppendObjectResult, error) { params := map[string]interface{}{} params["append"] = nil @@ -386,7 +383,11 @@ func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Opti listener := GetProgressListener(options) handleOptions(headers, opts) - resp, err := bucket.Client.Conn.Do("POST", bucket.BucketName, request.ObjectKey, params, headers, + + ctxArg, _ := FindOption(options, contextArg, nil) + ctx, _ := ctxArg.(context.Context) + + resp, err := bucket.Client.Conn.DoWithContext(ctx, "POST", bucket.BucketName, request.ObjectKey, params, headers, request.Reader, initCRC, listener) // get response header @@ -424,7 +425,6 @@ func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Opti // objectKey the object key to delete. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DeleteObject(objectKey string, options ...Option) error { params, _ := GetRawParams(options) resp, err := bucket.do("DELETE", objectKey, params, options, nil, nil) @@ -439,11 +439,11 @@ func (bucket Bucket) DeleteObject(objectKey string, options ...Option) error { // // objectKeys the object keys to delete. // options the options for deleting objects. -// Supported option is DeleteObjectsQuiet which means it will not return error even deletion failed (not recommended). By default it's not used. +// +// Supported option is DeleteObjectsQuiet which means it will not return error even deletion failed (not recommended). By default it's not used. // // DeleteObjectsResult the result object. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (DeleteObjectsResult, error) { out := DeleteObjectsResult{} dxml := deleteXML{} @@ -475,11 +475,11 @@ func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (Dele // // objectVersions the object keys and versions to delete. // options the options for deleting objects. -// Supported option is DeleteObjectsQuiet which means it will not return error even deletion failed (not recommended). By default it's not used. +// +// Supported option is DeleteObjectsQuiet which means it will not return error even deletion failed (not recommended). By default it's not used. // // DeleteObjectVersionsResult the result object. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DeleteObjectVersions(objectVersions []DeleteObject, options ...Option) (DeleteObjectVersionsResult, error) { out := DeleteObjectVersionsResult{} dxml := deleteXML{} @@ -506,7 +506,6 @@ func (bucket Bucket) DeleteObjectVersions(objectVersions []DeleteObject, options // // string the result response body. // error it's nil if no error, otherwise it's an error. -// func (bucket Bucket) DeleteMultipleObjectsXml(xmlData string, options ...Option) (string, error) { buffer := new(bytes.Buffer) bs := []byte(xmlData) @@ -534,7 +533,6 @@ func (bucket Bucket) DeleteMultipleObjectsXml(xmlData string, options ...Option) // bool flag of object's existence (true:exists; false:non-exist) when error is nil. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) IsObjectExist(objectKey string, options ...Option) (bool, error) { _, err := bucket.GetObjectMeta(objectKey, options...) if err == nil { @@ -554,23 +552,23 @@ func (bucket Bucket) IsObjectExist(objectKey string, options ...Option) (bool, e // ListObjects lists the objects under the current bucket. // // options it contains all the filters for listing objects. -// It could specify a prefix filter on object keys, the max keys count to return and the object key marker and the delimiter for grouping object names. -// The key marker means the returned objects' key must be greater than it in lexicographic order. // -// For example, if the bucket has 8 objects, my-object-1, my-object-11, my-object-2, my-object-21, -// my-object-22, my-object-3, my-object-31, my-object-32. If the prefix is my-object-2 (no other filters), then it returns -// my-object-2, my-object-21, my-object-22 three objects. If the marker is my-object-22 (no other filters), then it returns -// my-object-3, my-object-31, my-object-32 three objects. If the max keys is 5, then it returns 5 objects. -// The three filters could be used together to achieve filter and paging functionality. -// If the prefix is the folder name, then it could list all files under this folder (including the files under its subfolders). -// But if the delimiter is specified with '/', then it only returns that folder's files (no subfolder's files). The direct subfolders are in the commonPrefixes properties. -// For example, if the bucket has three objects fun/test.jpg, fun/movie/001.avi, fun/movie/007.avi. And if the prefix is "fun/", then it returns all three objects. -// But if the delimiter is '/', then only "fun/test.jpg" is returned as files and fun/movie/ is returned as common prefix. +// It could specify a prefix filter on object keys, the max keys count to return and the object key marker and the delimiter for grouping object names. +// The key marker means the returned objects' key must be greater than it in lexicographic order. // -// For common usage scenario, check out sample/list_object.go. +// For example, if the bucket has 8 objects, my-object-1, my-object-11, my-object-2, my-object-21, +// my-object-22, my-object-3, my-object-31, my-object-32. If the prefix is my-object-2 (no other filters), then it returns +// my-object-2, my-object-21, my-object-22 three objects. If the marker is my-object-22 (no other filters), then it returns +// my-object-3, my-object-31, my-object-32 three objects. If the max keys is 5, then it returns 5 objects. +// The three filters could be used together to achieve filter and paging functionality. +// If the prefix is the folder name, then it could list all files under this folder (including the files under its subfolders). +// But if the delimiter is specified with '/', then it only returns that folder's files (no subfolder's files). The direct subfolders are in the commonPrefixes properties. +// For example, if the bucket has three objects fun/test.jpg, fun/movie/001.avi, fun/movie/007.avi. And if the prefix is "fun/", then it returns all three objects. +// But if the delimiter is '/', then only "fun/test.jpg" is returned as files and fun/movie/ is returned as common prefix. // -// ListObjectsResult the return value after operation succeeds (only valid when error is nil). +// For common usage scenario, check out sample/list_object.go. // +// ListObjectsResult the return value after operation succeeds (only valid when error is nil). func (bucket Bucket) ListObjects(options ...Option) (ListObjectsResult, error) { var out ListObjectsResult @@ -653,10 +651,10 @@ func (bucket Bucket) ListObjectVersions(options ...Option) (ListObjectVersionsRe // // objectKey object // options options for setting the metadata. The valid options are CacheControl, ContentDisposition, ContentEncoding, Expires, -// ServerSideEncryption, and custom metadata. // -// error it's nil if no error, otherwise it's an error object. +// ServerSideEncryption, and custom metadata. // +// error it's nil if no error, otherwise it's an error object. func (bucket Bucket) SetObjectMeta(objectKey string, options ...Option) error { options = append(options, MetadataDirective(MetaReplace)) _, err := bucket.CopyObject(objectKey, objectKey, options...) @@ -667,11 +665,11 @@ func (bucket Bucket) SetObjectMeta(objectKey string, options ...Option) error { // // objectKey object key. // options the constraints of the object. Only when the object meets the requirements this method will return the metadata. Otherwise returns error. Valid options are IfModifiedSince, IfUnmodifiedSince, -// IfMatch, IfNoneMatch. For more details check out https://www.alibabacloud.com/help/en/object-storage-service/latest/headobject +// +// IfMatch, IfNoneMatch. For more details check out https://www.alibabacloud.com/help/en/object-storage-service/latest/headobject // // http.Header object meta when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectDetailedMeta(objectKey string, options ...Option) (http.Header, error) { params, _ := GetRawParams(options) resp, err := bucket.do("HEAD", objectKey, params, options, nil, nil) @@ -692,7 +690,6 @@ func (bucket Bucket) GetObjectDetailedMeta(objectKey string, options ...Option) // // http.Header the object's metadata, valid when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectMeta(objectKey string, options ...Option) (http.Header, error) { params, _ := GetRawParams(options) params["objectMeta"] = nil @@ -721,7 +718,6 @@ func (bucket Bucket) GetObjectMeta(objectKey string, options ...Option) (http.He // objectAcl object ACL. Valid options are PrivateACL, PublicReadACL, PublicReadWriteACL. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType, options ...Option) error { options = append(options, ObjectACL(objectACL)) params, _ := GetRawParams(options) @@ -740,7 +736,6 @@ func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType, options . // // GetObjectACLResult the result object when error is nil. GetObjectACLResult.Acl is the object ACL. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectACL(objectKey string, options ...Option) (GetObjectACLResult, error) { var out GetObjectACLResult params, _ := GetRawParams(options) @@ -767,7 +762,6 @@ func (bucket Bucket) GetObjectACL(objectKey string, options ...Option) (GetObjec // targetObjectKey the target object key to point to. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) PutSymlink(symObjectKey string, targetObjectKey string, options ...Option) error { options = append(options, symlinkTarget(url.QueryEscape(targetObjectKey))) params, _ := GetRawParams(options) @@ -786,8 +780,8 @@ func (bucket Bucket) PutSymlink(symObjectKey string, targetObjectKey string, opt // objectKey the symlink object's key. // // error it's nil if no error, otherwise it's an error object. -// When error is nil, the target file key is in the X-Oss-Symlink-Target header of the returned object. // +// When error is nil, the target file key is in the X-Oss-Symlink-Target header of the returned object. func (bucket Bucket) GetSymlink(objectKey string, options ...Option) (http.Header, error) { params, _ := GetRawParams(options) params["symlink"] = nil @@ -817,7 +811,6 @@ func (bucket Bucket) GetSymlink(objectKey string, options ...Option) (http.Heade // objectKey object key to restore. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) RestoreObject(objectKey string, options ...Option) error { params, _ := GetRawParams(options) params["restore"] = nil @@ -888,7 +881,6 @@ func (bucket Bucket) RestoreObjectXML(objectKey, configXML string, options ...Op // // string returns the signed URL, when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) SignURL(objectKey string, method HTTPMethod, expiredInSec int64, options ...Option) (string, error) { err := CheckObjectName(objectKey) if err != nil { @@ -920,11 +912,11 @@ func (bucket Bucket) SignURL(objectKey string, method HTTPMethod, expiredInSec i // signedURL signed URL. // reader io.Reader the read instance for reading the data for the upload. // options the options for uploading the data. The valid options are CacheControl, ContentDisposition, ContentEncoding, -// Expires, ServerSideEncryption, ObjectACL and custom metadata. Check out the following link for details: -// https://www.alibabacloud.com/help/en/object-storage-service/latest/putobject // -// error it's nil if no error, otherwise it's an error object. +// Expires, ServerSideEncryption, ObjectACL and custom metadata. Check out the following link for details: +// https://www.alibabacloud.com/help/en/object-storage-service/latest/putobject // +// error it's nil if no error, otherwise it's an error object. func (bucket Bucket) PutObjectWithURL(signedURL string, reader io.Reader, options ...Option) error { resp, err := bucket.DoPutObjectWithURL(signedURL, reader, options) if err != nil { @@ -943,7 +935,6 @@ func (bucket Bucket) PutObjectWithURL(signedURL string, reader io.Reader, option // options options for uploading, same as the options in PutObject function. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) PutObjectFromFileWithURL(signedURL, filePath string, options ...Option) error { fd, err := os.Open(filePath) if err != nil { @@ -968,7 +959,6 @@ func (bucket Bucket) PutObjectFromFileWithURL(signedURL, filePath string, option // // Response the response object which contains the HTTP response. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DoPutObjectWithURL(signedURL string, reader io.Reader, options []Option) (*Response, error) { listener := GetProgressListener(options) @@ -994,12 +984,12 @@ func (bucket Bucket) DoPutObjectWithURL(signedURL string, reader io.Reader, opti // // signedURL the signed URL. // options options for downloading the object. Valid options are IfModifiedSince, IfUnmodifiedSince, IfMatch, -// IfNoneMatch, AcceptEncoding. For more information, check out the following link: -// https://www.alibabacloud.com/help/en/object-storage-service/latest/getobject +// +// IfNoneMatch, AcceptEncoding. For more information, check out the following link: +// https://www.alibabacloud.com/help/en/object-storage-service/latest/getobject // // io.ReadCloser the reader object for getting the data from response. It needs be closed after the usage. It's only valid when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectWithURL(signedURL string, options ...Option) (io.ReadCloser, error) { result, err := bucket.DoGetObjectWithURL(signedURL, options) if err != nil { @@ -1015,7 +1005,6 @@ func (bucket Bucket) GetObjectWithURL(signedURL string, options ...Option) (io.R // options the options for downloading object. Check out the parameter options in function GetObject for the reference. // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) GetObjectToFileWithURL(signedURL, filePath string, options ...Option) error { tempFilePath := filePath + TempFileSuffix @@ -1066,7 +1055,6 @@ func (bucket Bucket) GetObjectToFileWithURL(signedURL, filePath string, options // // GetObjectResult the result object when the error is nil. // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) DoGetObjectWithURL(signedURL string, options []Option) (*GetObjectResult, error) { params, _ := GetRawParams(options) resp, err := bucket.doURL("GET", signedURL, params, options, nil, nil) @@ -1096,18 +1084,15 @@ func (bucket Bucket) DoGetObjectWithURL(signedURL string, options []Option) (*Ge return result, nil } -// // ProcessObject apply process on the specified image file. // // The supported process includes resize, rotate, crop, watermark, format, // udf, customized style, etc. // -// // objectKey object key to process. // process process string, such as "image/resize,w_100|sys/saveas,o_dGVzdC5qcGc,b_dGVzdA" // // error it's nil if no error, otherwise it's an error object. -// func (bucket Bucket) ProcessObject(objectKey string, process string, options ...Option) (ProcessObjectResult, error) { var out ProcessObjectResult params, _ := GetRawParams(options) @@ -1124,6 +1109,35 @@ func (bucket Bucket) ProcessObject(objectKey string, process string, options ... return out, err } +// +// AsyncProcessObject apply async process on the specified image file. +// +// The supported process includes resize, rotate, crop, watermark, format, +// udf, customized style, etc. +// +// +// objectKey object key to process. +// asyncProcess process string, such as "image/resize,w_100|sys/saveas,o_dGVzdC5qcGc,b_dGVzdA" +// +// error it's nil if no error, otherwise it's an error object. +// +func (bucket Bucket) AsyncProcessObject(objectKey string, asyncProcess string, options ...Option) (AsyncProcessObjectResult, error) { + var out AsyncProcessObjectResult + params, _ := GetRawParams(options) + params["x-oss-async-process"] = nil + processData := fmt.Sprintf("%v=%v", "x-oss-async-process", asyncProcess) + data := strings.NewReader(processData) + + resp, err := bucket.do("POST", objectKey, params, nil, data, nil) + if err != nil { + return out, err + } + defer resp.Body.Close() + + err = jsonUnmarshal(resp.Body, &out) + return out, err +} + // // PutObjectTagging add tagging to object // @@ -1131,7 +1145,6 @@ func (bucket Bucket) ProcessObject(objectKey string, process string, options ... // tagging tagging to be added // // error nil if success, otherwise error -// func (bucket Bucket) PutObjectTagging(objectKey string, tagging Tagging, options ...Option) error { bs, err := xml.Marshal(tagging) if err != nil { @@ -1175,13 +1188,11 @@ func (bucket Bucket) GetObjectTagging(objectKey string, options ...Option) (GetO return out, err } -// // DeleteObjectTagging delete object taggging // // objectKey object key to delete tagging // // error nil if success, otherwise error -// func (bucket Bucket) DeleteObjectTagging(objectKey string, options ...Option) error { params, _ := GetRawParams(options) params["tagging"] = nil @@ -1224,7 +1235,11 @@ func (bucket Bucket) doInner(method, objectName string, params map[string]interf if len(bucket.BucketName) > 0 && err != nil { return nil, err } - resp, err := bucket.Client.Conn.Do(method, bucket.BucketName, objectName, + + ctxArg, _ := FindOption(options, contextArg, nil) + ctx, _ := ctxArg.(context.Context) + + resp, err := bucket.Client.Conn.DoWithContext(ctx, method, bucket.BucketName, objectName, params, headers, data, 0, listener) // get response header @@ -1252,13 +1267,17 @@ func (bucket Bucket) do(method, objectName string, params map[string]interface{} func (bucket Bucket) doURL(method HTTPMethod, signedURL string, params map[string]interface{}, options []Option, data io.Reader, listener ProgressListener) (*Response, error) { + headers := make(map[string]string) err := handleOptions(headers, options) if err != nil { return nil, err } - resp, err := bucket.Client.Conn.DoURL(method, signedURL, headers, data, 0, listener) + ctxArg, _ := FindOption(options, contextArg, nil) + ctx, _ := ctxArg.(context.Context) + + resp, err := bucket.Client.Conn.DoURLWithContext(ctx, method, signedURL, headers, data, 0, listener) // get response header respHeader, _ := FindOption(options, responseHeader, nil) diff --git a/oss/bucket_test.go b/oss/bucket_test.go index 9901f96..76fe819 100644 --- a/oss/bucket_test.go +++ b/oss/bucket_test.go @@ -2,6 +2,7 @@ package oss import ( "bytes" + "context" "crypto/hmac" "crypto/sha1" "crypto/tls" @@ -2279,6 +2280,182 @@ func (s *OssBucketSuite) TestGetConfig(c *C) { c.Assert(bucket.GetConfig().IsEnableMD5, Equals, false) } +func (s *OssBucketSuite) TestForcePathStyle(c *C) { + url, err := url.ParseRequestURI(endpoint) + client, err := New(endpoint, accessID, accessKey, ForcePathStyle(true)) + c.Assert(err, IsNil) + + _, err = client.GetBucketInfo(bucketName) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).Code, Equals, "SecondLevelDomainForbidden") + c.Assert(err.(ServiceError).HostID, Equals, url.Host) + + bucket, err := client.Bucket(bucketName) + c.Assert(err, IsNil) + + c.Assert(bucket.GetConfig().IsPathStyle, Equals, true) + + objectName := "demo.txt" + + err = bucket.PutObject(objectName, strings.NewReader("hi oss")) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).Code, Equals, "SecondLevelDomainForbidden") + + str, err := bucket.SignURL(objectName, HTTPPut, 3600) + c.Assert(err, IsNil) + strUrl := endpoint + "/" + bucketName + "/" + objectName + c.Assert(strings.Contains(str, strUrl), Equals, true) +} + +func (s *OssBucketSuite) TestUseCname(c *C) { + url, err := url.ParseRequestURI(endpoint) + c.Assert(err, IsNil) + cnameEndpoint := bucketName + "." + url.Host + client, err := New(cnameEndpoint, accessID, accessKey, UseCname(true)) + c.Assert(err, IsNil) + + info, err := client.GetBucketInfo(bucketName) + + c.Assert(err, IsNil) + c.Assert(info.BucketInfo.Name, Equals, bucketName) + + client, err = New(cnameEndpoint, accessID, accessKey) + _, err = client.GetBucketInfo(bucketName) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).HostID, Equals, bucketName+"."+cnameEndpoint) +} + +func (s *OssBucketSuite) TestContextTimeout(c *C) { + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + bucket, err := client.Bucket(bucketName) + c.Assert(err, IsNil) + + objectName := objectNamePrefix + RandStr(8) + objectValue := "红藕香残玉簟秋。轻解罗裳，独上兰舟。云中谁寄锦书来？雁字回时，月满西楼。" + + bucketNameDest := bucketNamePrefix + RandLowStr(8) + c.Assert(err, IsNil) + + objectNameDest := objectName + RandLowStr(5) + + // Put + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 1*time.Nanosecond) + defer cancel() + err = bucket.PutObject(objectName, strings.NewReader(objectValue), WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + + // Get not exist object + _, err = bucket.GetObject(objectName) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).StatusCode, Equals, 404) + + err = s.bucket.PutObject(objectName, strings.NewReader(objectValue)) + c.Assert(err, IsNil) + + // Get + _, err = bucket.GetObject(objectName, WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + + // CopyObjectTo + _, err = bucket.CopyObjectTo(bucketNameDest, objectNameDest, objectName, WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + // Delete + err = s.bucket.DeleteObject(objectName) + c.Assert(err, IsNil) + + var nextPos int64 + // String append + nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader("红藕香残玉簟秋。轻解罗裳，独上兰舟。"), nextPos, WithContext(ctx)) + c.Assert(err, NotNil) + + // Put with URL + signedURL, err := bucket.SignURL(objectName, HTTPPut, 3600) + c.Assert(err, IsNil) + + err = bucket.PutObjectWithURL(signedURL, strings.NewReader(objectValue), WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + + // Get not exist object + _, err = bucket.GetObject(objectName) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).StatusCode, Equals, 404) + + err = s.bucket.PutObject(objectName, strings.NewReader(objectValue)) + c.Assert(err, IsNil) + + // Get with URL + signedURL, err = bucket.SignURL(objectName, HTTPGet, 3600) + c.Assert(err, IsNil) + + _, err = bucket.GetObjectWithURL(signedURL, WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + + err = s.bucket.DeleteObject(objectName) + c.Assert(err, IsNil) + + content := RandStr(1024 * 1024) + var fileName = "test-file-" + RandStr(8) + CreateFile(fileName, content, c) + + fd, err := os.Open(fileName) + c.Assert(err, IsNil) + defer fd.Close() + + err = bucket.PutObjectFromFile(objectName, fileName, WithContext(ctx)) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + + os.Remove(fileName) +} + +func (s *OssBucketSuite) TestContextTimeoutBigFile(c *C) { + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + bucket, err := client.Bucket(bucketName) + c.Assert(err, IsNil) + + objectName := objectNamePrefix + RandStr(8) + + //big file + content := RandStr(5 * 1024 * 1024) + var fileName = "test-file-" + RandStr(8) + CreateFile(fileName, content, c) + + fd, err := os.Open(fileName) + c.Assert(err, IsNil) + defer fd.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + objectNameBigfile := objectName + "-bigfile" + err = bucket.PutObjectFromFile(objectNameBigfile, fileName, WithContext(ctx)) + if ctx != nil { + select { + case <-ctx.Done(): + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "context deadline exceeded"), Equals, true) + default: + c.Fail() + } + } + + // Get not exist object + _, err = bucket.GetObject(objectNameBigfile) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).StatusCode, Equals, 404) + + os.Remove(fileName) +} + func (s *OssBucketSuite) TestSTSToken(c *C) { objectName := objectNamePrefix + RandStr(8) objectValue := "红藕香残玉簟秋。轻解罗裳，独上兰舟。云中谁寄锦书来？雁字回时，月满西楼。" @@ -2648,6 +2825,37 @@ func (s *OssBucketSuite) TestProcessObject(c *C) { c.Assert(err, NotNil) } +// TestAsyncProcessObject +func (s *OssBucketSuite) TestAsyncProcessObject(c *C) { + videoUrl := "https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/video.mp4?spm=a2c4g.64555.0.0.515675979u4B8w&file=video.mp4" + fileName := "video.mp4" + // 发起get请求 + resp, err := http.Get(videoUrl) + c.Assert(err, IsNil) + defer resp.Body.Close() + + // 创建文件 + file, err := os.Create(fileName) + c.Assert(err, IsNil) + defer file.Close() + _, err = io.Copy(file, resp.Body) + c.Assert(err, IsNil) + + err = s.bucket.PutObjectFromFile("demo.avi", fileName) + c.Assert(err, IsNil) + + sourceImageName := "demo.avi" + style := "video/convert,f_avi,vcodec_h265,s_1920x1080,vb_2000000,fps_30,acodec_aac,ab_100000,sn_1" + targetObject := "demo" + process := fmt.Sprintf("%s|sys/saveas,b_%v,o_%v", style, strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(bucketName)), "="), strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(targetObject)), "=")) + _, err = s.bucket.AsyncProcessObject(sourceImageName, process) + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).Code, Equals, "Imm Client") + c.Assert(strings.Contains(err.(ServiceError).Message, "ResourceNotFound, The specified resource Attachment is not found"), Equals, true) + + os.Remove(fileName) +} + // Private func CreateFileAndWrite(fileName string, data []byte) error { os.Remove(fileName) diff --git a/oss/client.go b/oss/client.go index 30aeccb..96d68ee 100644 --- a/oss/client.go +++ b/oss/client.go @@ -49,10 +49,6 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption) // URL parse url := &urlMaker{} - err := url.Init(config.Endpoint, config.IsCname, config.IsUseProxy) - if err != nil { - return nil, err - } // HTTP connect conn := &Conn{config: config, url: url} @@ -68,6 +64,11 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption) option(client) } + err := url.InitExt(config.Endpoint, config.IsCname, config.IsUseProxy, config.IsPathStyle) + if err != nil { + return nil, err + } + if config.AuthVersion != AuthV1 && config.AuthVersion != AuthV2 && config.AuthVersion != AuthV4 { return nil, fmt.Errorf("Init client Error, invalid Auth version: %v", config.AuthVersion) } @@ -493,9 +494,32 @@ func (client Client) SetBucketReferer(bucketName string, referrers []string, all if err != nil { return err } - buffer := new(bytes.Buffer) - buffer.Write(bs) + return client.PutBucketRefererXml(bucketName, string(bs), options...) +} + +// SetBucketRefererV2 gets the bucket's referer white list. +// +// setBucketReferer SetBucketReferer bucket referer config in struct format. +// +// GetBucketRefererResponse the result object upon successful request. It's only valid when error is nil. +// error it's nil if no error, otherwise it's an error object. +// +func (client Client) SetBucketRefererV2(bucketName string, setBucketReferer RefererXML, options ...Option) error { + bs, err := xml.Marshal(setBucketReferer) + if err != nil { + return err + } + return client.PutBucketRefererXml(bucketName, string(bs), options...) +} + +// PutBucketRefererXml set bucket's style +// bucketName the bucket name. +// xmlData the style in xml format +// error it's nil if no error, otherwise it's an error object. +func (client Client) PutBucketRefererXml(bucketName, xmlData string, options ...Option) error { + buffer := new(bytes.Buffer) + buffer.Write([]byte(xmlData)) contentType := http.DetectContentType(buffer.Bytes()) headers := map[string]string{} headers[HTTPHeaderContentType] = contentType @@ -511,24 +535,33 @@ func (client Client) SetBucketReferer(bucketName string, referrers []string, all } // GetBucketReferer gets the bucket's referrer white list. -// // bucketName the bucket name. -// -// GetBucketRefererResponse the result object upon successful request. It's only valid when error is nil. +// GetBucketRefererResult the result object upon successful request. It's only valid when error is nil. // error it's nil if no error, otherwise it's an error object. -// func (client Client) GetBucketReferer(bucketName string, options ...Option) (GetBucketRefererResult, error) { var out GetBucketRefererResult + body, err := client.GetBucketRefererXml(bucketName, options...) + if err != nil { + return out, err + } + err = xmlUnmarshal(strings.NewReader(body), &out) + return out, err +} + +// GetBucketRefererXml gets the bucket's referrer white list. +// bucketName the bucket name. +// GetBucketRefererResponse the bucket referer config result in xml format. +// error it's nil if no error, otherwise it's an error object. +func (client Client) GetBucketRefererXml(bucketName string, options ...Option) (string, error) { params := map[string]interface{}{} params["referer"] = nil resp, err := client.do("GET", bucketName, params, nil, nil, options...) if err != nil { - return out, err + return "", err } defer resp.Body.Close() - - err = xmlUnmarshal(resp.Body, &out) - return out, err + body, err := ioutil.ReadAll(resp.Body) + return string(body), err } // SetBucketLogging sets the bucket logging settings. @@ -2100,6 +2133,9 @@ func (client Client) GetBucketReplicationProgress(bucketName string, ruleId stri func (client Client) GetBucketAccessMonitor(bucketName string, options ...Option) (GetBucketAccessMonitorResult, error) { var out GetBucketAccessMonitorResult body, err := client.GetBucketAccessMonitorXml(bucketName, options...) + if err != nil { + return out, err + } err = xmlUnmarshal(strings.NewReader(body), &out) return out, err } @@ -2363,6 +2399,9 @@ func (client Client) PutBucketResourceGroupXml(bucketName string, xmlData string func (client Client) GetBucketResourceGroup(bucketName string, options ...Option) (GetBucketResourceGroupResult, error) { var out GetBucketResourceGroupResult body, err := client.GetBucketResourceGroupXml(bucketName, options...) + if err != nil { + return out, err + } err = xmlUnmarshal(strings.NewReader(body), &out) return out, err } @@ -2424,6 +2463,9 @@ func (client Client) PutBucketStyleXml(bucketName, styleName, xmlData string, op func (client Client) GetBucketStyle(bucketName, styleName string, options ...Option) (GetBucketStyleResult, error) { var out GetBucketStyleResult body, err := client.GetBucketStyleXml(bucketName, styleName, options...) + if err != nil { + return out, err + } err = xmlUnmarshal(strings.NewReader(body), &out) return out, err } @@ -2454,6 +2496,9 @@ func (client Client) GetBucketStyleXml(bucketName, styleName string, options ... func (client Client) ListBucketStyle(bucketName string, options ...Option) (GetBucketListStyleResult, error) { var out GetBucketListStyleResult body, err := client.ListBucketStyleXml(bucketName, options...) + if err != nil { + return out, err + } err = xmlUnmarshal(strings.NewReader(body), &out) return out, err } @@ -2498,6 +2543,19 @@ func (client Client) DeleteBucketStyle(bucketName, styleName string, options ... func (client Client) GetBucketCallbackPolicy(bucketName string, options ...Option) (GetBucketCallbackPolicyResult, error) { var out GetBucketCallbackPolicyResult body, err := client.GetBucketCallbackPolicyXml(bucketName, options...) + if err != nil { + return out, err + } + err = xmlUnmarshal(strings.NewReader(body), &out) + return out, err +} + +// DescribeRegions get describe regions +// GetDescribeRegionsResult the result of bucket in xml format. +// error it's nil if no error, otherwise it's an error object. +func (client Client) DescribeRegions(options ...Option) (DescribeRegionsResult, error) { + var out DescribeRegionsResult + body, err := client.DescribeRegionsXml(options...) if err != nil { return out, err } @@ -2571,6 +2629,27 @@ func (client Client) PutBucketCallbackPolicyXml(bucketName, xmlData string, opti return CheckRespCode(resp.StatusCode, []int{http.StatusOK}) } +// DescribeRegionsXml get describe regions +// string the style result of bucket in xml format. +// error it's nil if no error, otherwise it's an error object. +func (client Client) DescribeRegionsXml(options ...Option) (string, error) { + params, err := GetRawParams(options) + if err != nil { + return "", err + } + if params["regions"] == nil { + params["regions"] = nil + } + resp, err := client.do("GET", "", params, nil, nil, options...) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + out := string(body) + return out, err +} + // LimitUploadSpeed set upload bandwidth limit speed,default is 0,unlimited // upSpeed KB/s, 0 is unlimited,default is 0 // error it's nil if success, otherwise failure @@ -2598,7 +2677,16 @@ func (client Client) LimitDownloadSpeed(downSpeed int) error { func UseCname(isUseCname bool) ClientOption { return func(client *Client) { client.Config.IsCname = isUseCname - client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) + } +} + +// ForcePathStyle sets the flag of using Path Style. By default it's false. +// +// isPathStyle true: the endpoint has the Path Style, false: the endpoint does not have Path Style. Default is false. +// +func ForcePathStyle(isPathStyle bool) ClientOption { + return func(client *Client) { + client.Config.IsPathStyle = isPathStyle } } @@ -2695,7 +2783,6 @@ func Proxy(proxyHost string) ClientOption { return func(client *Client) { client.Config.IsUseProxy = true client.Config.ProxyHost = proxyHost - client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) } } @@ -2712,7 +2799,6 @@ func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption { client.Config.IsAuthProxy = true client.Config.ProxyUser = proxyUser client.Config.ProxyPassword = proxyPassword - client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) } } diff --git a/oss/client_test.go b/oss/client_test.go index 781216c..73ab75f 100644 --- a/oss/client_test.go +++ b/oss/client_test.go @@ -369,6 +369,38 @@ func (s *OssClientSuite) TestCreateBucket(c *C) { } } +// TestCreateBucketWithServerEncryption +func (s *OssClientSuite) TestCreateBucketWithServerEncryption(c *C) { + var bucketNameTest = bucketNamePrefix + RandLowStr(6) + + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + // Create + client.DeleteBucket(bucketNameTest) + err = client.CreateBucket(bucketNameTest, ServerSideEncryption("KMS"), ServerSideDataEncryption("SM4")) + c.Assert(err, IsNil) + //sleep 3 seconds after create bucket + time.Sleep(timeoutInOperation) + + // verify bucket is exist + rs, err := client.GetBucketEncryption(bucketNameTest) + c.Assert(err, IsNil) + c.Assert(rs.SSEDefault.SSEAlgorithm, Equals, "KMS") + c.Assert(rs.SSEDefault.KMSDataEncryption, Equals, "SM4") + + client.DeleteBucket(bucketNameTest) + err = client.CreateBucket(bucketNameTest, ServerSideEncryption("AES256")) + c.Assert(err, IsNil) + + rs, err = client.GetBucketEncryption(bucketNameTest) + c.Assert(err, IsNil) + c.Assert(rs.SSEDefault.KMSDataEncryption, Equals, "") + c.Assert(rs.SSEDefault.SSEAlgorithm, Equals, "AES256") + + client.DeleteBucket(bucketNameTest) +} + func (s *OssClientSuite) TestCreateBucketRedundancyType(c *C) { bucketNameTest := bucketNamePrefix + RandLowStr(6) client, err := New(endpoint, accessID, accessKey) @@ -1378,6 +1410,85 @@ func (s *OssClientSuite) TestBucketLifecycleNegative(c *C) { c.Assert(err, NotNil) } +// TestBucketLifecycleWithFilterSize +func (s *OssClientSuite) TestBucketLifecycleWithFilterSize(c *C) { + var bucketNameTest = bucketNamePrefix + RandLowStr(6) + + greater := int64(500) + less := int64(645000) + filter := LifecycleFilter{ + ObjectSizeGreaterThan: &greater, + ObjectSizeLessThan: &less, + } + rule1 := LifecycleRule{ + ID: "rs1", + Prefix: "logs", + Status: "Enabled", + Transitions: []LifecycleTransition{ + { + Days: 30, + StorageClass: StorageIA, + }, + }, + Filter: &filter, + } + config4 := LifecycleConfiguration{ + Rules: []LifecycleRule{rule1}, + } + xmlData4, err := xml.Marshal(config4) + testLogger.Println(string(xmlData4)) + + rules := []LifecycleRule{rule1} + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + err = client.CreateBucket(bucketNameTest) + c.Assert(err, IsNil) + + err = client.SetBucketLifecycle(bucketNameTest, rules) + c.Assert(err, IsNil) + + _, err = client.GetBucketLifecycle(bucketNameTest) + c.Assert(err, IsNil) + + err = client.DeleteBucketLifecycle(bucketNameTest) + c.Assert(err, IsNil) + tag := Tag{Key: "key1", Value: "val1"} + filter2 := LifecycleFilter{ + ObjectSizeGreaterThan: &greater, + ObjectSizeLessThan: &less, + Not: []LifecycleFilterNot{ + { + Tag: &tag, + }, + }, + } + rule2 := LifecycleRule{ + ID: "rs2", + Prefix: "", + Status: "Enabled", + Transitions: []LifecycleTransition{ + { + Days: 30, + StorageClass: StorageIA, + }, + }, + Filter: &filter2, + } + rules2 := []LifecycleRule{rule2} + + err = client.SetBucketLifecycle(bucketNameTest, rules2) + c.Assert(err, IsNil) + + _, err = client.GetBucketLifecycle(bucketNameTest) + c.Assert(err, IsNil) + + err = client.DeleteBucketLifecycle(bucketNameTest) + err = client.DeleteBucket(bucketNameTest) + c.Assert(err, IsNil) + +} + // TestSetBucketReferer func (s *OssClientSuite) TestSetBucketReferer(c *C) { var bucketNameTest = bucketNamePrefix + RandLowStr(6) @@ -1441,6 +1552,89 @@ func (s *OssClientSuite) TestBucketRefererNegative(c *C) { testLogger.Println(err) } +// TestBucketRefererV2 +func (s *OssClientSuite) TestBucketRefererV2(c *C) { + var bucketNameTest = bucketNamePrefix + RandLowStr(6) + var referrers = []string{"http://www.aliyun.com", "https://www.aliyun.com"} + + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + err = client.CreateBucket(bucketNameTest) + c.Assert(err, IsNil) + time.Sleep(timeoutInOperation) + + res, err := client.GetBucketReferer(bucketNameTest) + c.Assert(res.AllowEmptyReferer, Equals, true) + c.Assert(len(res.RefererList), Equals, 0) + c.Assert(*res.AllowTruncateQueryString, Equals, true) + c.Assert(res.RefererBlacklist, IsNil) + + // Set referers + var set RefererXML + set.AllowEmptyReferer = false + set.RefererList = referrers + err = client.SetBucketRefererV2(bucketNameTest, set) + c.Assert(err, IsNil) + time.Sleep(timeoutInOperation) + + res, err = client.GetBucketReferer(bucketNameTest) + c.Assert(res.AllowEmptyReferer, Equals, false) + c.Assert(len(res.RefererList), Equals, 2) + c.Assert(res.RefererList[0], Equals, "http://www.aliyun.com") + c.Assert(res.RefererList[1], Equals, "https://www.aliyun.com") + + // Reset referer, referers empty + var del RefererXML + del.AllowEmptyReferer = true + del.RefererList = []string{} + err = client.SetBucketRefererV2(bucketNameTest, del) + c.Assert(err, IsNil) + + res, err = client.GetBucketReferer(bucketNameTest) + c.Assert(res.AllowEmptyReferer, Equals, true) + c.Assert(len(res.RefererList), Equals, 0) + + // Set referers + var setBucketReferer RefererXML + setBucketReferer.AllowEmptyReferer = false + setBucketReferer.RefererList = referrers + referer1 := "http://www.refuse.com" + referer2 := "https://*.hack.com" + referer3 := "http://ban.*.com" + referer4 := "https://www.?.deny.com" + setBucketReferer.RefererBlacklist = &RefererBlacklist{ + []string{ + referer1, referer2, referer3, referer4, + }, + } + setBucketReferer.AllowEmptyReferer = false + boolTrue := true + setBucketReferer.AllowTruncateQueryString = &boolTrue + err = client.SetBucketRefererV2(bucketNameTest, setBucketReferer) + c.Assert(err, IsNil) + time.Sleep(timeoutInOperation) + + res, err = client.GetBucketReferer(bucketNameTest) + c.Assert(res.AllowEmptyReferer, Equals, false) + c.Assert(len(res.RefererList), Equals, 2) + c.Assert(res.RefererList[0], Equals, "http://www.aliyun.com") + c.Assert(res.RefererList[1], Equals, "https://www.aliyun.com") + c.Assert(*res.AllowTruncateQueryString, Equals, true) + c.Assert(res.RefererBlacklist, NotNil) + c.Assert(len(res.RefererBlacklist.Referer), Equals, 4) + c.Assert(res.RefererBlacklist.Referer[0], Equals, referer1) + c.Assert(res.RefererBlacklist.Referer[3], Equals, referer4) + + del.AllowEmptyReferer = true + del.RefererList = []string{} + err = client.SetBucketRefererV2(bucketNameTest, del) + c.Assert(err, IsNil) + + err = client.DeleteBucket(bucketNameTest) + c.Assert(err, IsNil) +} + // TestSetBucketLogging func (s *OssClientSuite) TestSetBucketLogging(c *C) { var bucketNameTest = bucketNamePrefix + RandLowStr(6) @@ -2553,6 +2747,13 @@ func (s *OssClientSuite) TestHttpsEndpointProxy(c *C) { c.Assert(err, IsNil) } +func (s *OssBucketSuite) TestProxyNavigate(c *C) { + client, err := New(endpoint, accessID, accessKey, AuthProxy("http://127.0.0.1:8120", "user", "passwd")) + c.Assert(err, IsNil) + _, err = client.GetBucketInfo(bucketName) + c.Assert(strings.Contains(err.Error(), "proxyconnect tcp: dial tcp 127.0.0.1:8120"), Equals, true) +} + // Private func (s *OssClientSuite) checkBucket(buckets []BucketProperties, bucket string) bool { for _, v := range buckets { @@ -3449,6 +3650,71 @@ func (s *OssClientSuite) TestClientCredentialInfBuild(c *C) { c.Assert(err, IsNil) } +func (s *OssClientSuite) TestClientNewEnvironmentVariableCredentialsProvider(c *C) { + provider, err := ProviderWrongKeyId() + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "access key id is empty!") + provider, err = ProviderWrongSecret() + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "access key secret is empty!") + + os.Setenv("OSS_ACCESS_KEY_ID", accessID) + os.Setenv("OSS_ACCESS_KEY_SECRET", accessKey) + provider, err = NewEnvironmentVariableCredentialsProvider() + c.Assert(err, IsNil) + var bucketNameTest = bucketNamePrefix + RandLowStr(6) + client, err := New(endpoint, "", "", SetCredentialsProvider(&provider)) + c.Assert(err, IsNil) + err = client.CreateBucket(bucketNameTest) + c.Assert(err, IsNil) + err = client.DeleteBucket(bucketNameTest) + c.Assert(err, IsNil) +} + +func ProviderWrongKeyId() (EnvironmentVariableCredentialsProvider, error) { + var provider EnvironmentVariableCredentialsProvider + keyId := os.Getenv("OSS_ACCESS_KEY_ID_d") + if keyId == "" { + return provider, fmt.Errorf("access key id is empty!") + } + secret := os.Getenv("OSS_ACCESS_KEY_SECRET_d") + if accessKey == "" { + return provider, fmt.Errorf("access key secret is empty!") + } + token := os.Getenv("OSS_SESSION_TOKEN") + + envCredential := &envCredentials{ + AccessKeyId: keyId, + AccessKeySecret: secret, + SecurityToken: token, + } + return EnvironmentVariableCredentialsProvider{ + cred: envCredential, + }, nil +} + +func ProviderWrongSecret() (EnvironmentVariableCredentialsProvider, error) { + var provider EnvironmentVariableCredentialsProvider + keyId := os.Getenv("OSS_ACCESS_KEY_ID") + if keyId == "" { + return provider, fmt.Errorf("access key id is empty!") + } + secret := os.Getenv("OSS_ACCESS_KEY_SECRET_NOT_EXIST") + if secret == "" { + return provider, fmt.Errorf("access key secret is empty!") + } + token := os.Getenv("OSS_SESSION_TOKEN") + + envCredential := &envCredentials{ + AccessKeyId: keyId, + AccessKeySecret: secret, + SecurityToken: token, + } + return EnvironmentVariableCredentialsProvider{ + cred: envCredential, + }, nil +} + func (s *OssClientSuite) TestClientSetLocalIpError(c *C) { // create client and bucket ipAddr, err := net.ResolveIPAddr("ip", "127.0.0.1") @@ -5234,6 +5500,10 @@ func (s *OssClientSuite) TestBucketAccessMonitor(c *C) { c.Assert(err, IsNil) c.Assert(res.BucketInfo.AccessMonitor, Equals, "Disabled") + result, err := client.GetBucketAccessMonitor(bucketNameTest) + c.Assert(err, IsNil) + c.Assert(result.Status, Equals, "Disabled") + // Put Bucket Access Monitor access := PutBucketAccessMonitor{ Status: "Enabled", @@ -5256,7 +5526,7 @@ func (s *OssClientSuite) TestBucketAccessMonitor(c *C) { c.Assert(res.BucketInfo.AccessMonitor, Equals, "Enabled") // get bucket access monitor - result, err := client.GetBucketAccessMonitor(bucketNameTest) + result, err = client.GetBucketAccessMonitor(bucketNameTest) c.Assert(err, IsNil) c.Assert(result.Status, Equals, "Enabled") @@ -5470,6 +5740,11 @@ func (s *OssClientSuite) TestBucketStyle(c *C) { c.Assert(res.CreateTime != "", Equals, true) c.Assert(res.LastModifyTime != "", Equals, true) + _, err = client.GetBucketStyle(bucketNameTest, "no-exist-style") + c.Assert(err, NotNil) + c.Assert(err.(ServiceError).StatusCode, Equals, 404) + c.Assert(err.(ServiceError).Code, Equals, "NoSuchStyle") + style1 := "image/resize,w_200" styleName1 := "image-" + RandLowStr(6) err = client.PutBucketStyle(bucketNameTest, styleName1, style1) @@ -5506,6 +5781,31 @@ func (s *OssClientSuite) TestBucketStyle(c *C) { err = client.DeleteBucketStyle(bucketNameTest, styleName2) c.Assert(err, IsNil) + err = client.DeleteBucketStyle(bucketNameTest, "no-exist-style") + c.Assert(err, IsNil) + + list, err = client.ListBucketStyle(bucketNameTest) + c.Assert(err, IsNil) + c.Assert(len(list.Style), Equals, 0) + +} + +// TestDescribeRegions +func (s *OssClientSuite) TestDescribeRegions(c *C) { + client, err := New(endpoint, accessID, accessKey) + c.Assert(err, IsNil) + + list, err := client.DescribeRegions() + c.Assert(err, IsNil) + c.Assert(len(list.Regions) > 0, Equals, true) + + info, err := client.DescribeRegions(AddParam("regions", "oss-cn-hangzhou")) + c.Assert(err, IsNil) + c.Assert(len(info.Regions), Equals, 1) + c.Assert(info.Regions[0].Region, Equals, "oss-cn-hangzhou") + c.Assert(info.Regions[0].InternetEndpoint, Equals, "oss-cn-hangzhou.aliyuncs.com") + c.Assert(info.Regions[0].InternalEndpoint, Equals, "oss-cn-hangzhou-internal.aliyuncs.com") + c.Assert(info.Regions[0].AccelerateEndpoint, Equals, "oss-accelerate.aliyuncs.com") } // TestBucketCallbackPolicy diff --git a/oss/conf.go b/oss/conf.go index 94baef7..e569541 100644 --- a/oss/conf.go +++ b/oss/conf.go @@ -73,6 +73,68 @@ func (defBuild *defaultCredentialsProvider) GetCredentials() Credentials { return &defaultCredentials{config: defBuild.config} } +type envCredentials struct { + AccessKeyId string + AccessKeySecret string + SecurityToken string +} + +type EnvironmentVariableCredentialsProvider struct { + cred Credentials +} + +func (credentials *envCredentials) GetAccessKeyID() string { + return credentials.AccessKeyId +} + +func (credentials *envCredentials) GetAccessKeySecret() string { + return credentials.AccessKeySecret +} + +func (credentials *envCredentials) GetSecurityToken() string { + return credentials.SecurityToken +} + +func (defBuild *EnvironmentVariableCredentialsProvider) GetCredentials() Credentials { + var accessID, accessKey, token string + if defBuild.cred == nil { + accessID = os.Getenv("OSS_ACCESS_KEY_ID") + accessKey = os.Getenv("OSS_ACCESS_KEY_SECRET") + token = os.Getenv("OSS_SESSION_TOKEN") + } else { + accessID = defBuild.cred.GetAccessKeyID() + accessKey = defBuild.cred.GetAccessKeySecret() + token = defBuild.cred.GetSecurityToken() + } + + return &envCredentials{ + AccessKeyId: accessID, + AccessKeySecret: accessKey, + SecurityToken: token, + } +} + +func NewEnvironmentVariableCredentialsProvider() (EnvironmentVariableCredentialsProvider, error) { + var provider EnvironmentVariableCredentialsProvider + accessID := os.Getenv("OSS_ACCESS_KEY_ID") + if accessID == "" { + return provider, fmt.Errorf("access key id is empty!") + } + accessKey := os.Getenv("OSS_ACCESS_KEY_SECRET") + if accessKey == "" { + return provider, fmt.Errorf("access key secret is empty!") + } + token := os.Getenv("OSS_SESSION_TOKEN") + envCredential := &envCredentials{ + AccessKeyId: accessID, + AccessKeySecret: accessKey, + SecurityToken: token, + } + return EnvironmentVariableCredentialsProvider{ + cred: envCredential, + }, nil +} + // Config defines oss configuration type Config struct { Endpoint string // OSS endpoint @@ -84,6 +146,7 @@ type Config struct { Timeout uint // Timeout in seconds. By default it's 60. SecurityToken string // STS Token IsCname bool // If cname is in the endpoint. + IsPathStyle bool // If Path Style is in the endpoint. HTTPTimeout HTTPTimeout // HTTP timeout HTTPMaxConns HTTPMaxConns // Http max connections IsUseProxy bool // Flag of using proxy. @@ -194,6 +257,7 @@ func getDefaultOssConfig() *Config { config.Timeout = 60 // Seconds config.SecurityToken = "" config.IsCname = false + config.IsPathStyle = false config.HTTPTimeout.ConnectTimeout = time.Second * 30 // 30s config.HTTPTimeout.ReadWriteTimeout = time.Second * 60 // 60s diff --git a/oss/conn.go b/oss/conn.go index 1466150..3538538 100644 --- a/oss/conn.go +++ b/oss/conn.go @@ -2,6 +2,7 @@ package oss import ( "bytes" + "context" "crypto/md5" "encoding/base64" "encoding/json" @@ -49,7 +50,7 @@ var signKeyList = []string{"acl", "uploads", "location", "cors", "x-oss-enable-md5", "x-oss-enable-sha1", "x-oss-enable-sha256", "x-oss-hash-ctx", "x-oss-md5-ctx", "transferAcceleration", "regionList", "cloudboxes", "x-oss-ac-source-ip", "x-oss-ac-subnet-mask", "x-oss-ac-vpc-id", "x-oss-ac-forward-allow", - "metaQuery", "resourceGroup", "rtc", + "metaQuery", "resourceGroup", "rtc", "x-oss-async-process", } // init initializes Conn @@ -88,6 +89,12 @@ func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) // Do sends request and returns the response func (conn Conn) Do(method, bucketName, objectName string, params map[string]interface{}, headers map[string]string, + data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { + return conn.DoWithContext(nil, method, bucketName, objectName, params, headers, data, initCRC, listener) +} + +// DoWithContext sends request and returns the response with context +func (conn Conn) DoWithContext(ctx context.Context, method, bucketName, objectName string, params map[string]interface{}, headers map[string]string, data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { urlParams := conn.getURLParams(params) subResource := conn.getSubResource(params) @@ -100,11 +107,17 @@ func (conn Conn) Do(method, bucketName, objectName string, params map[string]int resource = conn.getResourceV4(bucketName, objectName, subResource) } - return conn.doRequest(method, uri, resource, headers, data, initCRC, listener) + return conn.doRequest(ctx, method, uri, resource, headers, data, initCRC, listener) } // DoURL sends the request with signed URL and returns the response result. func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]string, + data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { + return conn.DoURLWithContext(nil, method, signedURL, headers, data, initCRC, listener) +} + +// DoURLWithContext sends the request with signed URL and context and returns the response result. +func (conn Conn) DoURLWithContext(ctx context.Context, method HTTPMethod, signedURL string, headers map[string]string, data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { // Get URI from signedURL uri, err := url.ParseRequestURI(signedURL) @@ -123,6 +136,9 @@ func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]s Host: uri.Host, } + if ctx != nil { + req = req.WithContext(ctx) + } tracker := &readerTracker{completedBytes: 0} fd, crc := conn.handleBody(req, data, initCRC, listener, tracker) if fd != nil { @@ -158,9 +174,10 @@ func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]s resp, err := conn.client.Do(req) if err != nil { // Transfer failed + conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error()) event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0) publishProgress(listener, event) - conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error()) + return nil, err } @@ -280,10 +297,12 @@ func (conn Conn) getResourceV4(bucketName, objectName, subResource string) strin return fmt.Sprintf("/%s/%s", bucketName, subResource) } -func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string, headers map[string]string, +func (conn Conn) doRequest(ctx context.Context, method string, uri *url.URL, canonicalizedResource string, headers map[string]string, data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { method = strings.ToUpper(method) - req := &http.Request{ + var req *http.Request + var err error + req = &http.Request{ Method: method, URL: uri, Proto: "HTTP/1.1", @@ -292,7 +311,9 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st Header: make(http.Header), Host: uri.Host, } - + if ctx != nil { + req = req.WithContext(ctx) + } tracker := &readerTracker{completedBytes: 0} fd, crc := conn.handleBody(req, data, initCRC, listener, tracker) if fd != nil { @@ -341,10 +362,10 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st resp, err := conn.client.Do(req) if err != nil { + conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error()) // Transfer failed event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0) publishProgress(listener, event) - conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error()) return nil, err } @@ -787,9 +808,10 @@ func (c *timeoutConn) SetWriteDeadline(t time.Time) error { // UrlMaker builds URL and resource const ( - urlTypeCname = 1 - urlTypeIP = 2 - urlTypeAliyun = 3 + urlTypeCname = 1 + urlTypeIP = 2 + urlTypeAliyun = 3 + urlTypePathStyle = 4 ) type urlMaker struct { @@ -801,6 +823,11 @@ type urlMaker struct { // Init parses endpoint func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error { + return um.InitExt(endpoint, isCname, isProxy, false) +} + +// InitExt parses endpoint +func (um *urlMaker) InitExt(endpoint string, isCname bool, isProxy bool, isPathStyle bool) error { if strings.HasPrefix(endpoint, "http://") { um.Scheme = "http" um.NetLoc = endpoint[len("http://"):] @@ -833,6 +860,8 @@ func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error { um.Type = urlTypeIP } else if isCname { um.Type = urlTypeCname + } else if isPathStyle { + um.Type = urlTypePathStyle } else { um.Type = urlTypeAliyun } @@ -881,7 +910,7 @@ func (um urlMaker) buildURL(bucket, object string) (string, string) { if um.Type == urlTypeCname { host = um.NetLoc path = "/" + object - } else if um.Type == urlTypeIP { + } else if um.Type == urlTypeIP || um.Type == urlTypePathStyle { if bucket == "" { host = um.NetLoc path = "/" @@ -916,7 +945,7 @@ func (um urlMaker) buildURLV4(bucket, object string) (string, string) { if um.Type == urlTypeCname { host = um.NetLoc path = "/" + object - } else if um.Type == urlTypeIP { + } else if um.Type == urlTypeIP || um.Type == urlTypePathStyle { if bucket == "" { host = um.NetLoc path = "/" diff --git a/oss/conn_test.go b/oss/conn_test.go index be57867..b4ceedd 100644 --- a/oss/conn_test.go +++ b/oss/conn_test.go @@ -84,6 +84,22 @@ func (s *OssConnSuite) TestURLMarker(c *C) { c.Assert(um.Type, Equals, urlTypeIP) c.Assert(um.Scheme, Equals, "https") c.Assert(um.NetLoc, Equals, "[2401:b180::dc]:8080") + + um.InitExt("https://docs.github.com:8080", false, false, true) + c.Assert(um.Type, Equals, urlTypePathStyle) + c.Assert(um.Scheme, Equals, "https") + c.Assert(um.NetLoc, Equals, "docs.github.com:8080") + c.Assert(um.getURL("bucket", "object", "params").String(), Equals, "https://docs.github.com:8080/bucket/object?params") + c.Assert(um.getURL("", "object", "params").String(), Equals, "https://docs.github.com:8080/?params") + + um.InitExt("docs.github.com", false, false, true) + c.Assert(um.Type, Equals, urlTypePathStyle) + c.Assert(um.Scheme, Equals, "http") + c.Assert(um.NetLoc, Equals, "docs.github.com") + + c.Assert(um.getURL("bucket", "object", "params").String(), Equals, "http://docs.github.com/bucket/object?params") + c.Assert(um.getURL("bucket", "object", "").String(), Equals, "http://docs.github.com/bucket/object") + c.Assert(um.getURL("", "object", "").String(), Equals, "http://docs.github.com/") } func (s *OssConnSuite) TestAuth(c *C) { diff --git a/oss/const.go b/oss/const.go index 4b6922d..c675270 100644 --- a/oss/const.go +++ b/oss/const.go @@ -76,9 +76,12 @@ const ( // StorageColdArchive cold archive StorageColdArchive StorageClassType = "ColdArchive" + + // StorageDeepColdArchive deep cold archive + StorageDeepColdArchive StorageClassType = "DeepColdArchive" ) -//RedundancyType bucket data Redundancy type +// RedundancyType bucket data Redundancy type type DataRedundancyType string const ( @@ -89,7 +92,7 @@ const ( RedundancyZRS DataRedundancyType = "ZRS" ) -//ObjecthashFuncType +// ObjecthashFuncType type ObjecthashFuncType string const ( @@ -108,7 +111,7 @@ const ( BucketOwner PayerType = "BucketOwner" ) -//RestoreMode the restore mode for coldArchive object +// RestoreMode the restore mode for coldArchive object type RestoreMode string const ( @@ -242,7 +245,7 @@ const ( DefaultContentSha256 = "UNSIGNED-PAYLOAD" // for v4 signature - Version = "v2.2.7" // Go SDK version + Version = "v2.2.9" // Go SDK version ) // FrameType diff --git a/oss/option.go b/oss/option.go index ccae9f4..351c388 100644 --- a/oss/option.go +++ b/oss/option.go @@ -1,6 +1,7 @@ package oss import ( + "context" "fmt" "net/http" "net/url" @@ -12,9 +13,11 @@ import ( type optionType string const ( - optionParam optionType = "HTTPParameter" // URL parameter - optionHTTP optionType = "HTTPHeader" // HTTP header - optionArg optionType = "FuncArgument" // Function argument + optionParam optionType = "HTTPParameter" // URL parameter + optionHTTP optionType = "HTTPHeader" // HTTP header + optionContext optionType = "HTTPContext" // context + optionArg optionType = "FuncArgument" // Function argument + ) const ( @@ -27,6 +30,7 @@ const ( responseHeader = "x-response-header" redundancyType = "redundancy-type" objectHashFunc = "object-hash-func" + contextArg = "x-context-arg" ) type ( @@ -448,6 +452,11 @@ func ObjectHashFunc(value ObjecthashFuncType) Option { return addArg(objectHashFunc, value) } +// WithContext returns an option that sets the context for requests. +func WithContext(ctx context.Context) Option { + return addArg(contextArg, ctx) +} + // Checkpoint configuration type cpConfig struct { IsEnable bool diff --git a/oss/option_test.go b/oss/option_test.go index 5ed42bf..ac52dc6 100644 --- a/oss/option_test.go +++ b/oss/option_test.go @@ -1,7 +1,9 @@ package oss import ( + "context" "net/http" + "time" . "gopkg.in/check.v1" ) @@ -315,3 +317,20 @@ func (s *OssOptionSuite) TestDeleteOption(c *C) { c.Assert(str, Equals, "789") } + +func (s *OssOptionSuite) TestWithContext(c *C) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + options := []Option{WithContext(ctx)} + ctxArg, _ := FindOption(options, contextArg, nil) + + c.Assert(ctxArg, NotNil) + c.Assert(ctxArg.(context.Context), Equals, ctx) + + options = []Option{} + ctxArg, _ = FindOption(options, contextArg, nil) + c.Assert(ctxArg, Equals, nil) + ctxnil, _ := ctxArg.(context.Context) + c.Assert(ctxnil, Equals, nil) +} diff --git a/oss/type.go b/oss/type.go index 505acb6..4187f39 100644 --- a/oss/type.go +++ b/oss/type.go @@ -128,15 +128,17 @@ type LifecycleVersionTransition struct { // LifecycleFilter defines the rule's Filter propery type LifecycleFilter struct { - XMLName xml.Name `xml:"Filter"` - Not []LifecycleFilterNot `xml:"Not,omitempty"` + XMLName xml.Name `xml:"Filter"` + Not []LifecycleFilterNot `xml:"Not,omitempty"` + ObjectSizeGreaterThan *int64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan *int64 `xml:"ObjectSizeLessThan,omitempty"` } // LifecycleFilterNot defines the rule's Filter Not propery type LifecycleFilterNot struct { XMLName xml.Name `xml:"Not"` - Prefix string `xml:"Prefix,omitempty"` //Object prefix applicable to this exclusion rule - Tag *Tag `xml:"Tag,omitempty"` //the tags applicable to this exclusion rule + Prefix string `xml:"Prefix"` //Object prefix applicable to this exclusion rule + Tag *Tag `xml:"Tag,omitempty"` //the tags applicable to this exclusion rule } const iso8601DateFormat = "2006-01-02T15:04:05.000Z" @@ -206,14 +208,20 @@ type GetBucketLifecycleResult LifecycleConfiguration // RefererXML defines Referer configuration type RefererXML struct { - XMLName xml.Name `xml:"RefererConfiguration"` - AllowEmptyReferer bool `xml:"AllowEmptyReferer"` // Allow empty referrer - RefererList []string `xml:"RefererList>Referer"` // Referer whitelist + XMLName xml.Name `xml:"RefererConfiguration"` + AllowEmptyReferer bool `xml:"AllowEmptyReferer"` // Allow empty referrer + AllowTruncateQueryString *bool `xml:"AllowTruncateQueryString,omitempty"` + RefererList []string `xml:"RefererList>Referer"` // Referer whitelist + RefererBlacklist *RefererBlacklist `xml:"RefererBlacklist,omitempty"` // Referer blacklist } // GetBucketRefererResult defines result object for GetBucketReferer request type GetBucketRefererResult RefererXML +type RefererBlacklist struct { + Referer []string `xml:"Referer,omitempty"` +} + // LoggingXML defines logging configuration type LoggingXML struct { XMLName xml.Name `xml:"BucketLoggingStatus"` @@ -600,6 +608,13 @@ type ProcessObjectResult struct { Status string `json:"status"` } +// AsyncProcessObjectResult defines result object of AsyncProcessObject +type AsyncProcessObjectResult struct { + EventId string `json:"EventId"` + RequestId string `json:"RequestId"` + TaskId string `json:"TaskId"` +} + // decodeDeleteObjectsResult decodes deleting objects result in URL encoding func decodeDeleteObjectsResult(result *DeleteObjectVersionsResult) error { var err error @@ -1651,3 +1666,18 @@ type PolicyItem struct { Callback string `xml:"Callback"` CallbackVar string `xml:"CallbackVar"` } + +// DescribeRegionsResult define get the describe regions result +type DescribeRegionsResult RegionInfoList + +type RegionInfo struct { + Region string `xml:"Region"` + InternetEndpoint string `xml:"InternetEndpoint"` + InternalEndpoint string `xml:"InternalEndpoint"` + AccelerateEndpoint string `xml:"AccelerateEndpoint"` +} + +type RegionInfoList struct { + XMLName xml.Name `xml:"RegionInfoList"` + Regions []RegionInfo `xml:"RegionInfo"` +} diff --git a/oss/type_test.go b/oss/type_test.go index b8b18a8..b538421 100644 --- a/oss/type_test.go +++ b/oss/type_test.go @@ -3,11 +3,11 @@ package oss import ( "encoding/base64" "encoding/xml" + . "gopkg.in/check.v1" + "log" "net/url" "sort" "strings" - - . "gopkg.in/check.v1" ) type OssTypeSuite struct{} @@ -940,7 +940,7 @@ func (s *OssTypeSuite) TestLifeCycleRules(c *C) { c.Assert(err, IsNil) } -func (s *OssTypeSuite) TestLifeCycleRulesWithFilter(c *C) { +func (s *OssTypeSuite) TestGetBucketLifecycleResult(c *C) { xmlData := []byte(` @@ -1017,6 +1017,189 @@ func (s *OssTypeSuite) TestLifeCycleRulesWithFilter(c *C) { c.Assert(res1.Rules[0].Filter.Not[2].Prefix, Equals, "abc/not2/") c.Assert(res1.Rules[0].Filter.Not[2].Tag.Key, Equals, "notkey2") c.Assert(res1.Rules[0].Filter.Not[2].Tag.Value, Equals, "notvalue2") + + xmlData = []byte(` + + r1 + abc/ + + 500 + 64000 + + abc/not1/ + + notkey1 + notvalue1 + + + + abc/not2/ + + notkey2 + notvalue2 + + + + + + r2 + def/ + + 500 + + def/not1/ + + + def/not2/ + + notkey2 + notvalue2 + + + + + +`) + + var res2 GetBucketLifecycleResult + err = xml.Unmarshal(xmlData, &res2) + testLogger.Println(res2.Rules[1].Filter) + c.Assert(err, IsNil) + c.Assert(res2.Rules[0].ID, Equals, "r1") + c.Assert(res2.Rules[0].Prefix, Equals, "abc/") + c.Assert(*(res2.Rules[0].Filter.ObjectSizeGreaterThan), Equals, int64(500)) + c.Assert(*(res2.Rules[0].Filter.ObjectSizeLessThan), Equals, int64(64000)) + c.Assert(res2.Rules[0].Filter.Not[0].Prefix, Equals, "abc/not1/") + c.Assert(res2.Rules[0].Filter.Not[0].Tag.Key, Equals, "notkey1") + c.Assert(res2.Rules[0].Filter.Not[0].Tag.Value, Equals, "notvalue1") + + c.Assert(res2.Rules[0].Filter.Not[1].Prefix, Equals, "abc/not2/") + c.Assert(res2.Rules[0].Filter.Not[1].Tag.Key, Equals, "notkey2") + c.Assert(res2.Rules[0].Filter.Not[1].Tag.Value, Equals, "notvalue2") + + c.Assert(res2.Rules[1].ID, Equals, "r2") + c.Assert(res2.Rules[1].Prefix, Equals, "def/") + c.Assert(*(res2.Rules[1].Filter.ObjectSizeGreaterThan), Equals, int64(500)) + c.Assert(res2.Rules[1].Filter.ObjectSizeLessThan, IsNil) + c.Assert(res2.Rules[1].Filter.Not[0].Prefix, Equals, "def/not1/") + + c.Assert(res2.Rules[1].Filter.Not[1].Prefix, Equals, "def/not2/") + c.Assert(res2.Rules[1].Filter.Not[1].Tag.Key, Equals, "notkey2") + c.Assert(res2.Rules[1].Filter.Not[1].Tag.Value, Equals, "notvalue2") +} +func (s *OssTypeSuite) TestLifecycleConfiguration(c *C) { + expiration := LifecycleExpiration{ + Days: 30, + CreatedBeforeDate: "2015-11-11T00:00:00.000Z", + } + isTrue := true + isFalse := false + greater := int64(500) + less := int64(645000) + filter := LifecycleFilter{ + ObjectSizeGreaterThan: &greater, + ObjectSizeLessThan: &less, + } + rule0 := LifecycleRule{ + ID: "r0", + Prefix: "prefix0", + Status: "Enabled", + Expiration: &expiration, + } + rule1 := LifecycleRule{ + ID: "r1", + Prefix: "prefix1", + Status: "Enabled", + Expiration: &expiration, + Transitions: []LifecycleTransition{ + { + Days: 30, + StorageClass: StorageIA, + IsAccessTime: &isFalse, + }, + }, + Filter: &filter, + } + + abortMPU := LifecycleAbortMultipartUpload{ + Days: 30, + CreatedBeforeDate: "2015-11-11T00:00:00.000Z", + } + rule2 := LifecycleRule{ + ID: "r3", + Prefix: "prefix3", + Status: "Enabled", + Expiration: &expiration, + AbortMultipartUpload: &abortMPU, + NonVersionTransitions: []LifecycleVersionTransition{ + { + NoncurrentDays: 10, + StorageClass: StorageIA, + IsAccessTime: &isTrue, + ReturnToStdWhenVisit: &isFalse, + }, + }, + } + tag := Tag{ + Key: "key1", + Value: "val1", + } + filter3 := LifecycleFilter{ + ObjectSizeGreaterThan: &greater, + ObjectSizeLessThan: &less, + Not: []LifecycleFilterNot{ + { + Tag: &tag, + }, + }, + } + rule3 := LifecycleRule{ + ID: "r4", + Prefix: "", + Status: "Enabled", + Expiration: &expiration, + AbortMultipartUpload: &abortMPU, + NonVersionTransitions: []LifecycleVersionTransition{ + { + NoncurrentDays: 10, + StorageClass: StorageIA, + IsAccessTime: &isTrue, + ReturnToStdWhenVisit: &isFalse, + }, + }, + Filter: &filter3, + } + rules := []LifecycleRule{rule0, rule1, rule2, rule3} + config := LifecycleConfiguration{ + Rules: rules, + } + xmlData, err := xml.Marshal(config) + testLogger.Println(string(xmlData)) + c.Assert(err, IsNil) + c.Assert(string(xmlData), Equals, "r0prefix0Enabled302015-11-11T00:00:00.000Zr1prefix1Enabled302015-11-11T00:00:00.000Z30IAfalse500645000r3prefix3Enabled302015-11-11T00:00:00.000Z302015-11-11T00:00:00.000Z10IAtruefalser4Enabled302015-11-11T00:00:00.000Z302015-11-11T00:00:00.000Z10IAtruefalsekey1val1500645000") + + filter4 := LifecycleFilter{ + ObjectSizeGreaterThan: &greater, + ObjectSizeLessThan: &less, + Not: []LifecycleFilterNot{ + {}, + }, + } + rule4 := LifecycleRule{ + ID: "r5", + Prefix: "", + Status: "Enabled", + Expiration: &expiration, + Filter: &filter4, + } + rules4 := []LifecycleRule{rule4} + config4 := LifecycleConfiguration{ + Rules: rules4, + } + xmlData4, err := xml.Marshal(config4) + testLogger.Println(string(xmlData4)) + c.Assert(err, IsNil) + c.Assert(string(xmlData4), Equals, "r5Enabled302015-11-11T00:00:00.000Z500645000") } // Test Bucket Resource Group @@ -1584,5 +1767,159 @@ func (s *OssTypeSuite) TestPutBucketCallbackPolicy(c *C) { c.Assert(err, IsNil) testLogger.Println(string(bs)) c.Assert(string(bs), Equals, xmlData) +} + +func (s *OssTypeSuite) TestGetBucketRefererResult(c *C) { + xmlData := ` + + true + + +` + var repResult GetBucketRefererResult + err := xmlUnmarshal(strings.NewReader(xmlData), &repResult) + c.Assert(err, IsNil) + c.Assert(repResult.AllowEmptyReferer, Equals, true) + c.Assert(repResult.AllowTruncateQueryString, IsNil) + c.Assert(repResult.RefererList, IsNil) + c.Assert(repResult.RefererBlacklist, IsNil) + + xmlData = ` + +true +false + + http://www.aliyun.com + https://www.aliyun.com + http://www.*.com + https://www.?.aliyuncs.com + +` + + var repResult0 GetBucketRefererResult + err = xmlUnmarshal(strings.NewReader(xmlData), &repResult0) + c.Assert(err, IsNil) + c.Assert(repResult0.AllowEmptyReferer, Equals, true) + c.Assert(*repResult0.AllowTruncateQueryString, Equals, false) + c.Assert(len(repResult0.RefererList), Equals, 4) + c.Assert(repResult0.RefererList[1], Equals, "https://www.aliyun.com") + c.Assert(repResult0.RefererBlacklist, IsNil) + + xmlData = ` + + false + false + + http://www.aliyun.com + https://www.aliyun.com + http://www.*.com + https://www.?.aliyuncs.com + + + http://www.refuse.com + https://*.hack.com + http://ban.*.com + https://www.?.deny.com + +` + var repResult1 GetBucketRefererResult + err = xmlUnmarshal(strings.NewReader(xmlData), &repResult1) + c.Assert(err, IsNil) + c.Assert(repResult1.AllowEmptyReferer, Equals, false) + c.Assert(*repResult1.AllowTruncateQueryString, Equals, false) + c.Assert(len(repResult1.RefererList), Equals, 4) + c.Assert(repResult1.RefererList[3], Equals, "https://www.?.aliyuncs.com") + + c.Assert(len(repResult1.RefererList), Equals, 4) + c.Assert((repResult1.RefererBlacklist).Referer[2], Equals, "http://ban.*.com") +} + +func (s *OssTypeSuite) TestRefererXML(c *C) { + xmlData := `true` + var setBucketReferer RefererXML + setBucketReferer.AllowEmptyReferer = true + setBucketReferer.RefererList = []string{} + xmlBody, err := xml.Marshal(setBucketReferer) + log.Println(string(xmlBody)) + c.Assert(err, IsNil) + c.Assert(string(xmlBody), Equals, xmlData) + + xmlData1 := `truefalsehttp://www.aliyun.comhttps://www.aliyun.comhttp://www.*.comhttps://www.?.aliyuncs.com` + + setBucketReferer.AllowEmptyReferer = true + boolFalse := false + setBucketReferer.AllowTruncateQueryString = &boolFalse + setBucketReferer.RefererList = []string{ + "http://www.aliyun.com", + "https://www.aliyun.com", + "http://www.*.com", + "https://www.?.aliyuncs.com", + } + c.Log(setBucketReferer) + xmlBody, err = xml.Marshal(setBucketReferer) + c.Assert(err, IsNil) + c.Assert(string(xmlBody), Equals, xmlData1) + + xmlData2 := `falsefalsehttp://www.aliyun.comhttps://www.aliyun.comhttp://www.*.comhttps://www.?.aliyuncs.comhttp://www.refuse.comhttps://*.hack.comhttp://ban.*.comhttps://www.?.deny.com` + + setBucketReferer.AllowEmptyReferer = false + setBucketReferer.AllowTruncateQueryString = &boolFalse + setBucketReferer.RefererList = []string{ + "http://www.aliyun.com", + "https://www.aliyun.com", + "http://www.*.com", + "https://www.?.aliyuncs.com", + } + referer1 := "http://www.refuse.com" + referer2 := "https://*.hack.com" + referer3 := "http://ban.*.com" + referer4 := "https://www.?.deny.com" + setBucketReferer.RefererBlacklist = &RefererBlacklist{ + []string{ + referer1, referer2, referer3, referer4, + }, + } + xmlBody, err = xml.Marshal(setBucketReferer) + c.Assert(err, IsNil) + c.Assert(string(xmlBody), Equals, xmlData2) +} +func (s *OssTypeSuite) TestDescribeRegionsResult(c *C) { + xmlData := ` + + + oss-cn-hangzhou + oss-cn-hangzhou.aliyuncs.com + oss-cn-hangzhou-internal.aliyuncs.com + oss-accelerate.aliyuncs.com + + + oss-cn-shanghai + oss-cn-shanghai.aliyuncs.com + oss-cn-shanghai-internal.aliyuncs.com + oss-accelerate.aliyuncs.com + +` + var repResult DescribeRegionsResult + err := xmlUnmarshal(strings.NewReader(xmlData), &repResult) + c.Assert(err, IsNil) + c.Assert(repResult.Regions[0].Region, Equals, "oss-cn-hangzhou") + c.Assert(repResult.Regions[0].InternetEndpoint, Equals, "oss-cn-hangzhou.aliyuncs.com") + c.Assert(repResult.Regions[0].InternalEndpoint, Equals, "oss-cn-hangzhou-internal.aliyuncs.com") + c.Assert(repResult.Regions[0].AccelerateEndpoint, Equals, "oss-accelerate.aliyuncs.com") + + c.Assert(repResult.Regions[1].Region, Equals, "oss-cn-shanghai") + c.Assert(repResult.Regions[1].InternetEndpoint, Equals, "oss-cn-shanghai.aliyuncs.com") + c.Assert(repResult.Regions[1].InternalEndpoint, Equals, "oss-cn-shanghai-internal.aliyuncs.com") + c.Assert(repResult.Regions[1].AccelerateEndpoint, Equals, "oss-accelerate.aliyuncs.com") +} + +func (s *OssTypeSuite) TestAsyncProcessResult(c *C) { + jsonData := `{"EventId":"10C-1XqxdjCRx3x7gRim3go1yLUVWgm","RequestId":"B8AD6942-BBDE-571D-A9A9-525A4C34B2B3","TaskId":"MediaConvert-58a8f19f-697f-4f8d-ae2c-0d7b15bef68d"}` + var repResult AsyncProcessObjectResult + err := jsonUnmarshal(strings.NewReader(jsonData), &repResult) + c.Assert(err, IsNil) + c.Assert(repResult.EventId, Equals, "10C-1XqxdjCRx3x7gRim3go1yLUVWgm") + c.Assert(repResult.RequestId, Equals, "B8AD6942-BBDE-571D-A9A9-525A4C34B2B3") + c.Assert(repResult.TaskId, Equals, "MediaConvert-58a8f19f-697f-4f8d-ae2c-0d7b15bef68d") } diff --git a/sample.go b/sample.go index c5ccdf4..2867441 100644 --- a/sample.go +++ b/sample.go @@ -49,6 +49,7 @@ var sampleMap = map[string]interface{}{ "ObjectTaggingSample": sample.ObjectTaggingSample, "BucketEncryptionSample": sample.BucketEncryptionSample, "SelectObjectSample": sample.SelectObjectSample, + "DescribeRegionsSample": sample.DescribeRegionsSample, } func main() { diff --git a/sample/bucket_lifecycle.go b/sample/bucket_lifecycle.go index 4ac46ba..1a52fd8 100644 --- a/sample/bucket_lifecycle.go +++ b/sample/bucket_lifecycle.go @@ -113,10 +113,14 @@ func BucketLifecycleSample() { Key: "key1", Value: "value1", } + greater := int64(500) + less := int64(645000) filter := oss.LifecycleFilter{ + ObjectSizeLessThan: &greater, + ObjectSizeGreaterThan: &less, Not: []oss.LifecycleFilterNot{ { - Prefix: "logs1", + Prefix: "logs/log2", Tag: &tag, }, }, @@ -270,6 +274,28 @@ func BucketLifecycleSample() { true + + r1 + abc/ + + 500 + 64000 + + abc/not1/ + + notkey1 + notvalue1 + + + + abc/not2/ + + notkey2 + notvalue2 + + + + ` err = client.SetBucketLifecycleXml(bucketName, xmlData) @@ -341,6 +367,12 @@ func BucketLifecycleSample() { } if rule.Filter != nil { + if rule.Filter.ObjectSizeGreaterThan != nil { + fmt.Println("Lifecycle Rule Filter Object Size Greater Than:", *rule.Filter.ObjectSizeGreaterThan) + } + if rule.Filter.ObjectSizeLessThan != nil { + fmt.Println("Lifecycle Rule Filter Object Size Less Than:", *rule.Filter.ObjectSizeLessThan) + } for _, filterNot := range rule.Filter.Not { fmt.Println("Lifecycle Rule Filter Not Prefix:", filterNot.Prefix) if filterNot.Tag != nil { diff --git a/sample/bucket_referer.go b/sample/bucket_referer.go index 7d1e4db..2f1bc89 100644 --- a/sample/bucket_referer.go +++ b/sample/bucket_referer.go @@ -2,7 +2,6 @@ package sample import ( "fmt" - "github.com/aliyun/aliyun-oss-go-sdk/oss" ) @@ -39,19 +38,75 @@ func BucketRefererSample() { HandleError(err) } + // Case 3: Create Refer With SetBucketRefererV2 + var setBucketReferer oss.RefererXML + setBucketReferer.RefererList = []string{ + "http://www.aliyun.com", + "https://www.aliyun.com", + "http://www.???.aliyuncs.com", + "http://www.*.com", + } + referer1 := "http://www.refuse.com" + referer2 := "https://*.hack.com" + referer3 := "http://ban.*.com" + referer4 := "https://www.?.deny.com" + setBucketReferer.RefererBlacklist = &oss.RefererBlacklist{ + []string{ + referer1, referer2, referer3, referer4, + }, + } + setBucketReferer.AllowEmptyReferer = false + boolTrue := true + setBucketReferer.AllowTruncateQueryString = &boolTrue + err = client.SetBucketRefererV2(bucketName, setBucketReferer) + if err != nil { + HandleError(err) + } + + fmt.Println("Set Bucket Referer Success") + // Get bucket referer configuration - gbr, err := client.GetBucketReferer(bucketName) + refRes, err := client.GetBucketReferer(bucketName) + if err != nil { + HandleError(err) + } + fmt.Println("Allow Empty Referer: ", refRes.AllowEmptyReferer) + if refRes.AllowTruncateQueryString != nil { + fmt.Println("Allow Truncate QueryString: ", *refRes.AllowTruncateQueryString) + } + if len(refRes.RefererList) > 0 { + for _, referer := range refRes.RefererList { + fmt.Println("Referer List: ", referer) + } + } + if refRes.RefererBlacklist != nil { + for _, refererBlack := range refRes.RefererBlacklist.Referer { + fmt.Println("Referer Black List: ", refererBlack) + } + } + fmt.Println("Get Bucket Referer Success") + + // Delete bucket referer + // Case 1:Delete Refer With SetBucketReferer + err = client.SetBucketReferer(bucketName, []string{}, true) if err != nil { HandleError(err) } - fmt.Println("Bucket Referers:", gbr.RefererList, - "AllowEmptyReferer:", gbr.AllowEmptyReferer) + + // Case 2:Delete Refer With SetBucketRefererV2 + var delBucketReferer oss.RefererXML + delBucketReferer.RefererList = []string{} + delBucketReferer.AllowEmptyReferer = true + err = client.SetBucketRefererV2(bucketName, delBucketReferer) + if err != nil { + HandleError(err) + } + fmt.Println("Delete Bucket Referer Success") // Delete bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } - fmt.Println("BucketRefererSample completed") } diff --git a/sample/describe_regions.go b/sample/describe_regions.go new file mode 100644 index 0000000..0e5998b --- /dev/null +++ b/sample/describe_regions.go @@ -0,0 +1,45 @@ +package sample + +import ( + "fmt" + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +//DescribeRegionsSample shows how to get or list describe regions +func DescribeRegionsSample() { + // Create archive bucket + client, err := oss.New(endpoint, accessID, accessKey) + if err != nil { + HandleError(err) + } + + // Get describe regions + regionEndpoint := "oss-cn-hangzhou" + list, err := client.DescribeRegions(regionEndpoint) + if err != nil { + HandleError(err) + } + for _, region := range list.Regions { + fmt.Printf("Region:%s\n", region.Region) + fmt.Printf("Region Internet Endpoint:%s\n", region.InternetEndpoint) + fmt.Printf("Region Internal Endpoint:%s\n", region.InternalEndpoint) + fmt.Printf("Region Accelerate Endpoint:%s\n", region.AccelerateEndpoint) + } + fmt.Println("Get Describe Regions Success") + + // List describe regions + + list, err = client.DescribeRegions("") + if err != nil { + HandleError(err) + } + for _, region := range list.Regions { + fmt.Printf("Region:%s\n", region.Region) + fmt.Printf("Region Internet Endpoint:%s\n", region.InternetEndpoint) + fmt.Printf("Region Internal Endpoint:%s\n", region.InternalEndpoint) + fmt.Printf("Region Accelerate Endpoint:%s\n", region.AccelerateEndpoint) + } + fmt.Println("List Describe Regions Success") + + fmt.Println("DescribeRegionsSample completed") +}