From 666c0c2b66af3a4417eea066b09b5d92bbf23ab3 Mon Sep 17 00:00:00 2001 From: t-mawhite Date: Thu, 20 Jun 2024 14:24:35 -0700 Subject: [PATCH 1/2] new tests --- cmd/zt_traverser_blob_test.go | 1020 ++++++++++++++++++++++++++++++++- cmd/zt_traverser_type.go | 131 +++++ go.mod | 26 +- go.sum | 62 +- 4 files changed, 1193 insertions(+), 46 deletions(-) create mode 100644 cmd/zt_traverser_type.go diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index bc5ebc867..6ef030276 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -22,21 +22,25 @@ package cmd import ( "context" + "encoding/xml" "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" datalakedirectory "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/Azure/azure-storage-azcopy/v10/mock_server" "github.com/Azure/azure-storage-azcopy/v10/ste" "github.com/stretchr/testify/assert" + "strings" "testing" "time" ) -func TestIsSourceDirWithStub(t *testing.T) { +// original code that hits server +func TestIsSourceDirWithStub2(t *testing.T) { a := assert.New(t) bsc := getBlobServiceClient() @@ -61,6 +65,1020 @@ func TestIsSourceDirWithStub(t *testing.T) { a.Nil(err) } +func SetUpVariables() (accountname string, rawurl string, bloburl string, blobname string, containername string, credential *blob.SharedKeyCredential, err error) { + // Create a client + // Note: the key below is not a secret, this is the publicly documented Azurite key + accountName := "myfakeaccount" + accountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + rawURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + NewCredential, err := blob.NewSharedKeyCredential(accountName, accountKey) + + if err != nil { + return "", "", "", "", "", nil, err + } + + //set container and blob name + cName := generateContainerName() + bName := generateBlobName() + + // construct a string that points to blob name above + // https://accountname.blob.core.windows.net/containername/blobname + blobURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, cName, bName) + + return accountName, rawURL, blobURL, bName, cName, NewCredential, nil +} + +func TestIsSourceDirWithStub(t *testing.T) { + a := assert.New(t) + + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithHeader("x-ms-meta-"+common.POSIXFolderMeta, "true")) + + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + isDir, err := blobTraverser.IsDirectory(true) + a.True(isDir) + a.Nil(err) +} + +func TestIsDirFile(t *testing.T) { + a := assert.New(t) + + //create mock server (do every time) + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(200)) + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + isDir, err := blobTraverser.IsDirectory(true) + a.False(isDir) + a.Nil(err) +} + +func MockErrorBody(message string) string { + body := "" + body += "" + body += "" + message + "" + body += " " + body += "" + return body +} + +func TestIsDirFileError(t *testing.T) { + a := assert.New(t) + + //create mock server (do every time) + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(409), mock_server.WithHeader("x-ms-error-code", "BlobUsesCustomerSpecifiedEncryption"), mock_server.WithBody([]byte(MockErrorBody("BlobUsesCustomerSpecifiedEncryption")))) + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + isDir, err := blobTraverser.IsDirectory(true) + a.False(isDir) + a.Nil(err) +} + +func SetNoFileBody(accounturl string, containername string) string { + body := "" + body += "" + body += "string-value \n string-value \n 1 \n string-value \n \n \n " + return body +} +func TestIsDirNoFile(t *testing.T) { + a := assert.New(t) + + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + accountName, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //properties part + srv.AppendResponse(mock_server.WithStatusCode(404), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) + //list part + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetNoFileBody(accountName, cName)))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + isDir, err := blobTraverser.IsDirectory(true) + a.False(isDir) + a.Equal("The specified file was not found.", err.Error()) +} + +func SetBodyWithBlob(accounturl string, containername string) string { + body := "" + body += "" + body += "string-value \n string-value \n 3 \n string-value \n \n \n blob-name \n Sun, 27 Sep 2009 18:09:03 GMT \n Sun, 27 Sep 2009 18:09:03 GMT\n true\n true\n " + body += " \n \n value \n \n \n \n \n TagName\n TagValue\n \n \n \n \n \n \n blob-prefix \n \n \n \n" + return body +} + +func TestIsDirWithList(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + accountName, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //properties part + srv.AppendResponse(mock_server.WithStatusCode(404), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) + + //list part + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlob(accountName, cName)))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + isDir, err := blobTraverser.IsDirectory(true) + a.True(isDir) + a.Nil(err) +} + +// this function tests the Traverse function getPropertiesIfSingleBlob and responds with the BlobUsesCustomerSpecifiedEncryption error +func TestTraverseGetPropErrorCode(t *testing.T) { + a := assert.New(t) + + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(409), mock_server.WithHeader("x-ms-error-code", "BlobUsesCustomerSpecifiedEncryption"), mock_server.WithBody([]byte(MockErrorBody("BlobUsesCustomerSpecifiedEncryption")))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, []ObjectFilter{}) + a.Equal(map[string]StoredObject{}, seenFiles) + a.Equal("this blob uses customer provided encryption keys (CPK). At the moment, AzCopy does not support CPK-encrypted blobs. If you wish to make use of this blob, we recommend using one of the Azure Storage SDKs", err.Error()) +} + +// this function tests the Traverse function getPropertiesIfSingleBlob if status code is 403 and returns error +func TestTraverseGetPropStatusCode(t *testing.T) { + a := assert.New(t) + + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(403), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, []ObjectFilter{}) + a.Equal(map[string]StoredObject{}, seenFiles) + a.True(strings.Contains(err.Error(), "cannot list files due to reason")) +} + +func MockTagBody(tagk string, tagv string) string { + tagSet := []*container.BlobTag{to.Ptr(container.BlobTag{Key: to.Ptr(tagk), Value: to.Ptr(tagv)})} + tags := container.BlobTags{BlobTagSet: tagSet} + out, err := xml.Marshal(tags) + // check err + if err != nil { + return "" + } + body := string(out) + return body +} + +func TestTraverseGetBlobTags(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize necessary variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //test the get properties call + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithHeader("x-ms-meta-proptest", "proptestval")) + //test the blob tags call + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(MockTagBody("keytest", "valtest")))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles["testpath"] = storedObject + return nil + }, nil) + a.Nil(err) + a.True(seenFiles["testpath"].blobTags["keytest"] == "valtest") + a.True(seenFiles["testpath"].Metadata != nil) +} + +func TestTraverseNoBlobTags(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(200)) + //no tags but successful call + srv.AppendResponse(mock_server.WithStatusCode(200)) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles["testpath"] = storedObject + return nil + }, nil) + a.Nil(err) + a.Nil(seenFiles["testpath"].blobTags) +} + +// this test calls Traverse and calls Serial List, then returns an error when Next Page is called +func TestTraverseSerialListNextPageError(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(200)) + //no tags but successful call + srv.AppendResponse(mock_server.WithStatusCode(200)) + srv.AppendResponse(mock_server.WithStatusCode(400)) + + //changed includeversion to be true to make parallelListing false + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + //isDir, err := blobTraverser.IsDirectory(true) + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles["testpath"] = storedObject + return nil + }, nil) + a.True(strings.Contains(err.Error(), "cannot list blobs. Failed with error")) +} + +func SetBodyWithBlobNew(accounturl string, containername string) string { + testmetadata := map[string]*string{ + "key1": to.Ptr("value1"), + "key2": to.Ptr("value2"), + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("blob-name"), Properties: to.Ptr(blobProp), Metadata: testmetadata})} + segment := container.BlobFlatListSegment{BlobItems: blobitem} + resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), MaxResults: to.Ptr(int32(1))} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls Serial List, then successfully returns +func TestTraverseSerialListSuccess(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize necessary variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + srv.AppendResponse(mock_server.WithStatusCode(200)) + //no tags but successful call + srv.AppendResponse(mock_server.WithStatusCode(200)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlobNew(rawURL, cName)))) + + //create new blob traverser + //changed includeversion to be true to make parallelListing false + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.name] = storedObject + return nil + }, nil) + a.Nil(err) + a.Equal(seenFiles["blob-name"].name, "blob-name") + a.EqualValues(seenFiles["blob-name"].size, int64(2)) + a.Equal(*seenFiles["blob-name"].Metadata["key1"], "value1") +} + +func SetBodyWithMultipleBlobs(accounturl string, containername string, folderflag bool) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + metadatafolder := map[string]*string{"hdi_isfolder": to.Ptr("false")} + if folderflag { + metadatafolder["hdi_isfolder"] = to.Ptr("true") + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/"), Properties: to.Ptr(blobProp), Metadata: metadatafolder}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/file2.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + segment := container.BlobFlatListSegment{BlobItems: blobitem} + resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Serial List and has a Subdirectory w/ hdi_isfolder +func TestTraverseSerialListWithFolder(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithMultipleBlobs(rawURL, cName, true)))) + + //create new blob traverser + //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") +} + +// this test calls Traverse and calls a Serial List and has a Subdirectory without hdi_isfolder +func TestTraverseSerialListNoFolder(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithMultipleBlobs(rawURL, cName, false)))) + + //create new blob traverser + //changed includeversion to be true to make parallelListing false and includeDirectoryStubs to false so that hdi_isfolder is recognized + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 3) + a.Equal(*seenFiles["folder1/folder2/"].Metadata["hdi_isfolder"], "false") + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") +} + +func SetBodyNoSubDir(accounturl string, containername string) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + segment := container.BlobFlatListSegment{BlobItems: blobitem} + resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Serial List and has no subdirectory +func TestTraverseSerialListNoSubDir(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoSubDir(rawURL, cName)))) + + //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/"].Metadata["key1"], "val1") +} + +func SetBodyNoBlob(accounturl string, containername string) string { + segment := container.BlobFlatListSegment{} + resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Serial List and has no blobs listed +func TestTraverseSerialListNoBlob(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoBlob(rawURL, cName)))) + + //create new blob traverser + //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 0) +} + +func SetHierachicalBody(accounturl string, containername string, folderflag bool) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + metadatafolder := map[string]*string{"hdi_isfolder": to.Ptr("false")} + if folderflag { + metadatafolder["hdi_isfolder"] = to.Ptr("true") + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/"), Properties: to.Ptr(blobProp), Metadata: metadatafolder}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/file2.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + blobprefix := []*container.BlobPrefix{to.Ptr(container.BlobPrefix{Name: to.Ptr("folder1/folder2/")})} + segment := container.BlobHierarchyListSegment{BlobItems: blobitem, BlobPrefixes: blobprefix} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Parallel List and has a Subdirectory w/ hdi_isfolder +func TestTraverseParallelListWithMarker(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBody(rawURL, cName, true)))) + + //create new blob traverser + //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false + blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") +} + +// this test calls Traverse and calls a Parallel List and has a Subdirectory without hdi_isfolder +func TestTraverseParallelListNoMarker(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBody(rawURL, cName, false)))) + + //create new blob traverser + //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false + blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 3) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/"].Metadata["hdi_isfolder"], "false") +} + +func SetHierachicalBodyNoSubDir(accounturl string, containername string) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file2.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + segment := container.BlobHierarchyListSegment{BlobItems: blobitem} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Parallel List and has no Subdirectory +func TestTraverseParallelListNoSubDir(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyNoSubDir(rawURL, cName)))) + + //create new blob traverser + //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false + blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/file2.txt"].Metadata["key1"], "val1") +} + +// set two bodies where the first blobprefix becomes prefix for second call +func SetHierachicalBodyRecursive(accounturl string, containername string, folderflag bool, secondcall bool) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + metadatafolder := map[string]*string{"hdi_isfolder": to.Ptr("false")} + if folderflag { + metadatafolder["hdi_isfolder"] = to.Ptr("true") + } + if !secondcall { + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/"), Properties: to.Ptr(blobProp), Metadata: metadatafolder})} + blobprefix := []*container.BlobPrefix{to.Ptr(container.BlobPrefix{Name: to.Ptr("folder1/folder2/")})} + segment := container.BlobHierarchyListSegment{BlobItems: blobitem, BlobPrefixes: blobprefix} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody + } else { + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/file2.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + segment := container.BlobHierarchyListSegment{BlobItems: blobitem} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/folder2/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody + } +} + +// this test calls Traverse and calls a Parallel List with two responses with the second prefix set as the first blobprefix +func TestTraverseParallelListWithMarkerRecursive(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyRecursive(rawURL, cName, true, false)))) + //second body uses first prefix + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyRecursive(rawURL, cName, false, true)))) + + //create new blob traverser + //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized + //made recursive true + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") +} + +// set two bodies where the first blobprefix becomes prefix for second call +func SetHierachicalBodyBlobTags(accounturl string, containername string) string { + metadatatest := map[string]*string{ + "key1": to.Ptr("val1"), + } + blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} + blobtags := container.BlobTags{BlobTagSet: []*container.BlobTag{to.Ptr(container.BlobTag{Key: to.Ptr("tagval"), Value: to.Ptr("keyval")})}} + blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), BlobTags: to.Ptr(blobtags), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} + //blobprefix := []*container.BlobPrefix{to.Ptr(container.BlobPrefix{Name: to.Ptr("folder1/folder2/")})} + segment := container.BlobHierarchyListSegment{BlobItems: blobitem} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Parallel List with two responses with the second prefix set as the first blobprefix and sets properties to dir +func TestTraverseParallelListBlobTags(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyBlobTags(rawURL, cName)))) + + //create new blob traverser + //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive true + blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 2) + a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/folder2/"].Metadata["key1"], "val1") + a.Equal(seenFiles["folder1/file1.txt"].blobTags["tagval"], "keyval") +} + +func SetHierachicalBodyNothing(accounturl string, containername string) string { + segment := container.BlobHierarchyListSegment{} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) + return newbody +} + +// this test calls Traverse and calls a Parallel List with no blob item or blob prefix +func TestTraverseParallelListNothing(t *testing.T) { + a := assert.New(t) + //create mock server + srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) + defer close() + //initialize variables + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() + a.Nil(err) + + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, + &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: srv, //passing in mock server + }}) + a.Nil(err) + //define what mock server should return as response + //first case is not a blob + srv.AppendResponse(mock_server.WithStatusCode(400)) + //use marshalling to append body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyNothing(rawURL, cName)))) + + //create new blob traverser + blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) + + //test method and validate + seenFiles := make(map[string]StoredObject) + + err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { + seenFiles[storedObject.relativePath] = storedObject + return nil + }, nil) + a.Nil(err) + a.Len(seenFiles, 0) +} + func TestIsSourceDirWithNoStub(t *testing.T) { a := assert.New(t) bsc := getBlobServiceClient() diff --git a/cmd/zt_traverser_type.go b/cmd/zt_traverser_type.go new file mode 100644 index 000000000..3d9649794 --- /dev/null +++ b/cmd/zt_traverser_type.go @@ -0,0 +1,131 @@ +package cmd + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "time" +) + +// ListBlobsFlatSegmentResponse - An enumeration of blobs +type ListBlobsFlatSegmentResponse struct { + // REQUIRED + ContainerName *string `xml:"ContainerName,attr"` + + // REQUIRED + Segment *BlobFlatListSegment `xml:"Blobs"` + + // REQUIRED + ServiceEndpoint *string `xml:"ServiceEndpoint,attr"` + Marker *string `xml:"Marker"` + MaxResults *int32 `xml:"MaxResults"` + NextMarker *string `xml:"NextMarker"` + Prefix *string `xml:"Prefix"` +} + +type BlobFlatListSegment struct { + // REQUIRED + BlobItems []*BlobItem `xml:"Blob"` +} + +// BlobItem - An Azure Storage blob +type BlobItem struct { + // REQUIRED + Deleted *bool `xml:"Deleted"` + + // REQUIRED + Name *string `xml:"Name"` + + // REQUIRED; Properties of a blob + Properties *BlobProperties `xml:"Properties"` + + // REQUIRED + Snapshot *string `xml:"Snapshot"` + + // Blob tags + BlobTags *BlobTags `xml:"Tags"` + HasVersionsOnly *bool `xml:"HasVersionsOnly"` + IsCurrentVersion *bool `xml:"IsCurrentVersion"` + + // Dictionary of + Metadata map[string]*string `xml:"Metadata"` + + // Dictionary of + OrMetadata map[string]*string `xml:"OrMetadata"` + VersionID *string `xml:"VersionId"` +} + +// BlobProperties - Properties of a blob +type BlobProperties struct { + // REQUIRED + ETag *azcore.ETag `xml:"Etag"` + + // REQUIRED + LastModified *time.Time `xml:"Last-Modified"` + AccessTier *AccessTier `xml:"AccessTier"` + AccessTierChangeTime *time.Time `xml:"AccessTierChangeTime"` + AccessTierInferred *bool `xml:"AccessTierInferred"` + ArchiveStatus *ArchiveStatus `xml:"ArchiveStatus"` + BlobSequenceNumber *int64 `xml:"x-ms-blob-sequence-number"` + BlobType *BlobType `xml:"BlobType"` + CacheControl *string `xml:"Cache-Control"` + ContentDisposition *string `xml:"Content-Disposition"` + ContentEncoding *string `xml:"Content-Encoding"` + ContentLanguage *string `xml:"Content-Language"` + + // Size in bytes + ContentLength *int64 `xml:"Content-Length"` + ContentMD5 []byte `xml:"Content-MD5"` + ContentType *string `xml:"Content-Type"` + CopyCompletionTime *time.Time `xml:"CopyCompletionTime"` + CopyID *string `xml:"CopyId"` + CopyProgress *string `xml:"CopyProgress"` + CopySource *string `xml:"CopySource"` + CopyStatus *CopyStatusType `xml:"CopyStatus"` + CopyStatusDescription *string `xml:"CopyStatusDescription"` + CreationTime *time.Time `xml:"Creation-Time"` + CustomerProvidedKeySHA256 *string `xml:"CustomerProvidedKeySha256"` + DeletedTime *time.Time `xml:"DeletedTime"` + DestinationSnapshot *string `xml:"DestinationSnapshot"` + + // The name of the encryption scope under which the blob is encrypted. + EncryptionScope *string `xml:"EncryptionScope"` + ExpiresOn *time.Time `xml:"Expiry-Time"` + ImmutabilityPolicyExpiresOn *time.Time `xml:"ImmutabilityPolicyUntilDate"` + ImmutabilityPolicyMode *ImmutabilityPolicyMode `xml:"ImmutabilityPolicyMode"` + IncrementalCopy *bool `xml:"IncrementalCopy"` + IsSealed *bool `xml:"Sealed"` + LastAccessedOn *time.Time `xml:"LastAccessTime"` + LeaseDuration *LeaseDurationType `xml:"LeaseDuration"` + LeaseState *LeaseStateType `xml:"LeaseState"` + LeaseStatus *LeaseStatusType `xml:"LeaseStatus"` + LegalHold *bool `xml:"LegalHold"` + + // If an object is in rehydrate pending state then this header is returned with priority of rehydrate. Valid values are High + // and Standard. + RehydratePriority *RehydratePriority `xml:"RehydratePriority"` + RemainingRetentionDays *int32 `xml:"RemainingRetentionDays"` + ServerEncrypted *bool `xml:"ServerEncrypted"` + TagCount *int32 `xml:"TagCount"` +} + +type BlobTags struct { + // REQUIRED + BlobTagSet []*BlobTag `xml:"TagSet>Tag"` +} + +type BlobTag struct { + // REQUIRED + Key *string `xml:"Key"` + + // REQUIRED + Value *string `xml:"Value"` +} + +type BlobType string +type AccessTier string +type ArchiveStatus string +type CopyStatusType string +type ImmutabilityPolicyMode string +type LeaseDurationType string +type LeaseStateType string +type LeaseStatusType string +type RehydratePriority string diff --git a/go.mod b/go.mod index 3876d7110..995ea0af2 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/Azure/azure-storage-azcopy/v10 +replace github.com/Azure/azure-sdk-for-go/sdk/storage/azblob => github.com/mawhite-msft/azure-sdk-for-go/sdk/storage/azblob v0.0.0-20240614172713-ee454601024f + require ( cloud.google.com/go/storage v1.29.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 @@ -15,25 +17,25 @@ require ( github.com/minio/minio-go v6.0.14+incompatible github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.6 - github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/spf13/cobra v1.4.0 github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 github.com/wastore/keyctl v0.3.1 - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.20.0 google.golang.org/api v0.114.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) -require github.com/stretchr/testify v1.8.4 +require github.com/stretchr/testify v1.9.0 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 github.com/Azure/go-autorest/autorest/date v0.3.0 - golang.org/x/net v0.23.0 + golang.org/x/net v0.24.0 ) require ( @@ -41,22 +43,22 @@ require ( cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ini/ini v1.66.4 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang-jwt/jwt/v5 v5.2.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -65,7 +67,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index d9ff4031f..72e7acc72 100644 --- a/go.sum +++ b/go.sum @@ -11,16 +11,14 @@ cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXW cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1 h1:mkaGMgFkpDJVs7QUQrHvqEEpJFvoDrqGaHqMkywhGN0= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.1.1/go.mod h1:3S0vo7Y+O3Fjnnon5JXVrlG2IrfQkXasvKWB4OwX1lk= github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.1 h1:59Jk/lsLhtEMq6Zi7t6aWDFMman28m2O9K67IPAqqEI= @@ -37,8 +35,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= @@ -55,8 +53,6 @@ github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -66,8 +62,8 @@ github.com/go-ini/ini v1.66.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -108,10 +104,9 @@ github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d h1:WR9P3d5kJAzyQrNR github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d/go.mod h1:cHjYsnAnSckPDx8/H01Y+owD1hf2adLA6VRiw4guEbA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -120,6 +115,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= +github.com/mawhite-msft/azure-sdk-for-go/sdk/storage/azblob v0.0.0-20240614172713-ee454601024f h1:3zQdvhd9J4+5LqZ1I73k8NP3upq61twePPck6r44Zhs= +github.com/mawhite-msft/azure-sdk-for-go/sdk/storage/azblob v0.0.0-20240614172713-ee454601024f/go.mod h1:sJ8JTIjNrTgqRsLtI+rXVqyQY+M6Okq6OfqLfesKGBU= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -134,9 +131,9 @@ github.com/pkg/xattr v0.4.6/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVW github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= @@ -145,14 +142,15 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 h1:Uzh85j0tl46Sf2OOx1wDePSWkz3Eq8XdCFkLXqaX8Bg= github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807/go.mod h1:zI8umr7xnBSyT9ZJ8wn48RiQ0EWXo4xmYLNw9FQvC9w= github.com/wastore/keyctl v0.3.1 h1:wMkYW9y9jGbQ1ARBLGLwnDdbgrkbuSeuIQeHy+BZOU0= @@ -162,8 +160,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -178,8 +176,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= @@ -198,15 +196,15 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -247,10 +245,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From a84eae280ba248cf9b17a0b5901af67a620c997c Mon Sep 17 00:00:00 2001 From: t-mawhite Date: Fri, 21 Jun 2024 11:54:40 -0700 Subject: [PATCH 2/2] completed traverser tests --- cmd/zt_traverser_blob_test.go | 183 ++++++++++++++-------------------- cmd/zt_traverser_type.go | 131 ------------------------ 2 files changed, 76 insertions(+), 238 deletions(-) delete mode 100644 cmd/zt_traverser_type.go diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index 6ef030276..ee5faf720 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -88,6 +88,7 @@ func SetUpVariables() (accountname string, rawurl string, bloburl string, blobna return accountName, rawURL, blobURL, bName, cName, NewCredential, nil } +// this test calls IsDirectory with the header hdi_isfolder to return true func TestIsSourceDirWithStub(t *testing.T) { a := assert.New(t) @@ -117,6 +118,7 @@ func TestIsSourceDirWithStub(t *testing.T) { a.Nil(err) } +// this test calls isDirectory mocking a file and isDirStub returns false func TestIsDirFile(t *testing.T) { a := assert.New(t) @@ -148,14 +150,10 @@ func TestIsDirFile(t *testing.T) { } func MockErrorBody(message string) string { - body := "" - body += "" - body += "" + message + "" - body += " " - body += "" - return body + return "" + message + " " } +// this test sends the error code "BlobUsesCustomerSpecifiedEncryption" and returns false func TestIsDirFileError(t *testing.T) { a := assert.New(t) @@ -174,7 +172,7 @@ func TestIsDirFileError(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response + //mock response has "BlobUsesCustomerSpecifiedEncryption" error srv.AppendResponse(mock_server.WithStatusCode(409), mock_server.WithHeader("x-ms-error-code", "BlobUsesCustomerSpecifiedEncryption"), mock_server.WithBody([]byte(MockErrorBody("BlobUsesCustomerSpecifiedEncryption")))) //create new blob traverser blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) @@ -185,12 +183,7 @@ func TestIsDirFileError(t *testing.T) { a.Nil(err) } -func SetNoFileBody(accounturl string, containername string) string { - body := "" - body += "" - body += "string-value \n string-value \n 1 \n string-value \n \n \n " - return body -} +// this test calls isDirectory and fails with get properties and also fails with NewListBlobs call by returning no blobs func TestIsDirNoFile(t *testing.T) { a := assert.New(t) @@ -209,10 +202,10 @@ func TestIsDirNoFile(t *testing.T) { }}) a.Nil(err) - //properties part + //get properties returns error srv.AppendResponse(mock_server.WithStatusCode(404), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) - //list part - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetNoFileBody(accountName, cName)))) + //list returns no blobs + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoBlob(accountName, cName)))) //create new blob traverser blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) @@ -223,14 +216,7 @@ func TestIsDirNoFile(t *testing.T) { a.Equal("The specified file was not found.", err.Error()) } -func SetBodyWithBlob(accounturl string, containername string) string { - body := "" - body += "" - body += "string-value \n string-value \n 3 \n string-value \n \n \n blob-name \n Sun, 27 Sep 2009 18:09:03 GMT \n Sun, 27 Sep 2009 18:09:03 GMT\n true\n true\n " - body += " \n \n value \n \n \n \n \n TagName\n TagValue\n \n \n \n \n \n \n blob-prefix \n \n \n \n" - return body -} - +// this test calls isDirectory and fails with get properties but returns blobs and passes with NewListBlobs call func TestIsDirWithList(t *testing.T) { a := assert.New(t) //create mock server @@ -248,10 +234,9 @@ func TestIsDirWithList(t *testing.T) { }}) a.Nil(err) - //properties part + //get properties returns error srv.AppendResponse(mock_server.WithStatusCode(404), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) - - //list part + //list returns blob items srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlob(accountName, cName)))) //create new blob traverser @@ -263,7 +248,7 @@ func TestIsDirWithList(t *testing.T) { a.Nil(err) } -// this function tests the Traverse function getPropertiesIfSingleBlob and responds with the BlobUsesCustomerSpecifiedEncryption error +// this test calls Traverse and getPropertiesIfSingleBlob and responds with the BlobUsesCustomerSpecifiedEncryption error func TestTraverseGetPropErrorCode(t *testing.T) { a := assert.New(t) @@ -282,7 +267,7 @@ func TestTraverseGetPropErrorCode(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response + //the mock server returns "BlobUsesCustomerSpecifiedEncryption" srv.AppendResponse(mock_server.WithStatusCode(409), mock_server.WithHeader("x-ms-error-code", "BlobUsesCustomerSpecifiedEncryption"), mock_server.WithBody([]byte(MockErrorBody("BlobUsesCustomerSpecifiedEncryption")))) //create new blob traverser @@ -299,7 +284,7 @@ func TestTraverseGetPropErrorCode(t *testing.T) { a.Equal("this blob uses customer provided encryption keys (CPK). At the moment, AzCopy does not support CPK-encrypted blobs. If you wish to make use of this blob, we recommend using one of the Azure Storage SDKs", err.Error()) } -// this function tests the Traverse function getPropertiesIfSingleBlob if status code is 403 and returns error +// this tests if the Traverse function getPropertiesIfSingleBlob status code is 403 and returns error func TestTraverseGetPropStatusCode(t *testing.T) { a := assert.New(t) @@ -318,7 +303,7 @@ func TestTraverseGetPropStatusCode(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response + //the mock server responds with a status code error srv.AppendResponse(mock_server.WithStatusCode(403), mock_server.WithHeader("x-ms-error-code", "BlobNotFound"), mock_server.WithBody([]byte(MockErrorBody("BlobNotFound")))) //create new blob traverser @@ -335,11 +320,10 @@ func TestTraverseGetPropStatusCode(t *testing.T) { a.True(strings.Contains(err.Error(), "cannot list files due to reason")) } -func MockTagBody(tagk string, tagv string) string { - tagSet := []*container.BlobTag{to.Ptr(container.BlobTag{Key: to.Ptr(tagk), Value: to.Ptr(tagv)})} +func MockTagBody(TagKey string, TagVal string) string { + tagSet := []*container.BlobTag{to.Ptr(container.BlobTag{Key: to.Ptr(TagKey), Value: to.Ptr(TagVal)})} tags := container.BlobTags{BlobTagSet: tagSet} out, err := xml.Marshal(tags) - // check err if err != nil { return "" } @@ -347,6 +331,7 @@ func MockTagBody(tagk string, tagv string) string { return body } +// this test calls Traverse and mocks a response with blob tags that ensures the set tags are in the stored object func TestTraverseGetBlobTags(t *testing.T) { a := assert.New(t) //create mock server @@ -383,6 +368,7 @@ func TestTraverseGetBlobTags(t *testing.T) { a.True(seenFiles["testpath"].Metadata != nil) } +// this test sends a blob response with no blob tags and checks that the stored object blob tags is nil func TestTraverseNoBlobTags(t *testing.T) { a := assert.New(t) //create mock server @@ -390,7 +376,7 @@ func TestTraverseNoBlobTags(t *testing.T) { defer close() //initialize variables - _, rawURL, blobURL, _, _, credential, err := SetUpVariables() + _, rawURL, blobURL, _, cName, credential, err := SetUpVariables() a.Nil(err) client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, @@ -403,7 +389,7 @@ func TestTraverseNoBlobTags(t *testing.T) { //define what mock server should return as response srv.AppendResponse(mock_server.WithStatusCode(200)) //no tags but successful call - srv.AppendResponse(mock_server.WithStatusCode(200)) + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlob(rawURL, cName)))) //create new blob traverser blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) @@ -441,13 +427,13 @@ func TestTraverseSerialListNextPageError(t *testing.T) { srv.AppendResponse(mock_server.WithStatusCode(200)) //no tags but successful call srv.AppendResponse(mock_server.WithStatusCode(200)) + //next page error srv.AppendResponse(mock_server.WithStatusCode(400)) //changed includeversion to be true to make parallelListing false blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) //test method and validate - //isDir, err := blobTraverser.IsDirectory(true) seenFiles := make(map[string]StoredObject) err = blobTraverser.Traverse(nil, func(storedObject StoredObject) error { @@ -457,7 +443,7 @@ func TestTraverseSerialListNextPageError(t *testing.T) { a.True(strings.Contains(err.Error(), "cannot list blobs. Failed with error")) } -func SetBodyWithBlobNew(accounturl string, containername string) string { +func SetBodyWithBlob(accounturl string, containername string) string { testmetadata := map[string]*string{ "key1": to.Ptr("value1"), "key2": to.Ptr("value2"), @@ -475,7 +461,7 @@ func SetBodyWithBlobNew(accounturl string, containername string) string { return newbody } -// this test calls Traverse and calls Serial List, then successfully returns +// this test calls Traverse with Serial List, then successfully reads a mocked blob body and stores as stored object func TestTraverseSerialListSuccess(t *testing.T) { a := assert.New(t) //create mock server @@ -498,7 +484,7 @@ func TestTraverseSerialListSuccess(t *testing.T) { //no tags but successful call srv.AppendResponse(mock_server.WithStatusCode(200)) //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlobNew(rawURL, cName)))) + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithBlob(rawURL, cName)))) //create new blob traverser //changed includeversion to be true to make parallelListing false @@ -537,7 +523,7 @@ func SetBodyWithMultipleBlobs(accounturl string, containername string, folderfla return newbody } -// this test calls Traverse and calls a Serial List and has a Subdirectory w/ hdi_isfolder +// this test calls Traverse with a Serial List and has a Subdirectory w/ hdi_isfolder that is skipped func TestTraverseSerialListWithFolder(t *testing.T) { a := assert.New(t) //create mock server @@ -560,8 +546,7 @@ func TestTraverseSerialListWithFolder(t *testing.T) { //use marshalling to append body srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithMultipleBlobs(rawURL, cName, true)))) - //create new blob traverser - //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized + //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so hdi_isfolder is recognized blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -577,7 +562,7 @@ func TestTraverseSerialListWithFolder(t *testing.T) { a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") } -// this test calls Traverse and calls a Serial List and has a Subdirectory without hdi_isfolder +// this test calls Traverse with a Serial List and has a Subdirectory without hdi_isfolder that is stored func TestTraverseSerialListNoFolder(t *testing.T) { a := assert.New(t) //create mock server @@ -600,7 +585,6 @@ func TestTraverseSerialListNoFolder(t *testing.T) { //use marshalling to append body srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyWithMultipleBlobs(rawURL, cName, false)))) - //create new blob traverser //changed includeversion to be true to make parallelListing false and includeDirectoryStubs to false so that hdi_isfolder is recognized blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) @@ -617,24 +601,36 @@ func TestTraverseSerialListNoFolder(t *testing.T) { a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") } -func SetBodyNoSubDir(accounturl string, containername string) string { +func SetBodyNoSubDir(accounturl string, containername string, flat bool) string { metadatatest := map[string]*string{ "key1": to.Ptr("val1"), } blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} - segment := container.BlobFlatListSegment{BlobItems: blobitem} - resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} + if flat { + segment := container.BlobFlatListSegment{BlobItems: blobitem} + resp := container.ListBlobsFlatSegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} + out, err := xml.Marshal(resp) + if err != nil { + fmt.Printf(err.Error()) + } + body := string(out) + newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + return newbody + } + //Hierarchical case + segment := container.BlobHierarchyListSegment{BlobItems: blobitem} + resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} out, err := xml.Marshal(resp) if err != nil { fmt.Printf(err.Error()) } body := string(out) - newbody := strings.Replace(body, "ListBlobsFlatSegmentResponse", "EnumerationResults", -1) + newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) return newbody } -// this test calls Traverse and calls a Serial List and has no subdirectory +// this test calls Traverse with a Serial List where there is no subdirectory func TestTraverseSerialListNoSubDir(t *testing.T) { a := assert.New(t) //create mock server @@ -652,13 +648,12 @@ func TestTraverseSerialListNoSubDir(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoSubDir(rawURL, cName)))) + //use marshalling to append body with no subdirectory + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoSubDir(rawURL, cName, true)))) - //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized + //changed includeversion to be true to make parallelListing false blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -704,14 +699,11 @@ func TestTraverseSerialListNoBlob(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body + //use marshalling to append body with no blob srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoBlob(rawURL, cName)))) - //create new blob traverser - //changed includeversion to be true to make parallelListing false and changed includeDirectoryStubs to false so that hdi_isfolder is recognized blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, true, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -725,7 +717,7 @@ func TestTraverseSerialListNoBlob(t *testing.T) { a.Len(seenFiles, 0) } -func SetHierachicalBody(accounturl string, containername string, folderflag bool) string { +func SetHierarchicalBody(accounturl string, containername string, folderflag bool) string { metadatatest := map[string]*string{ "key1": to.Ptr("val1"), } @@ -765,14 +757,13 @@ func TestTraverseParallelListWithMarker(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBody(rawURL, cName, true)))) + //use marshalling to append hierarchical body with folder flag + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBody(rawURL, cName, true)))) //create new blob traverser - //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false + //changed includeversion to be false to make parallelListing true, changed includeDirectoryStubs to false so that hdi_isfolder is recognized, and made recursive false blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -809,11 +800,10 @@ func TestTraverseParallelListNoMarker(t *testing.T) { //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBody(rawURL, cName, false)))) + //use marshalling to append hierarchical body with no folder flag + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBody(rawURL, cName, false)))) - //create new blob traverser - //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false + //changed includeversion to be false to make parallelListing true, changed includeDirectoryStubs to false so that hdi_isfolder is recognized, and made recursive false blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -830,24 +820,7 @@ func TestTraverseParallelListNoMarker(t *testing.T) { a.Equal(*seenFiles["folder1/folder2/"].Metadata["hdi_isfolder"], "false") } -func SetHierachicalBodyNoSubDir(accounturl string, containername string) string { - metadatatest := map[string]*string{ - "key1": to.Ptr("val1"), - } - blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} - blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file2.txt"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} - segment := container.BlobHierarchyListSegment{BlobItems: blobitem} - resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} - out, err := xml.Marshal(resp) - if err != nil { - fmt.Printf(err.Error()) - } - body := string(out) - newbody := strings.Replace(body, "ListBlobsHierarchySegmentResponse", "EnumerationResults", -1) - return newbody -} - -// this test calls Traverse and calls a Parallel List and has no Subdirectory +// this test calls Traverse with a Parallel List that has no Subdirectory func TestTraverseParallelListNoSubDir(t *testing.T) { a := assert.New(t) //create mock server @@ -869,7 +842,7 @@ func TestTraverseParallelListNoSubDir(t *testing.T) { //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyNoSubDir(rawURL, cName)))) + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetBodyNoSubDir(rawURL, cName, false)))) //create new blob traverser //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive false @@ -885,11 +858,11 @@ func TestTraverseParallelListNoSubDir(t *testing.T) { a.Nil(err) a.Len(seenFiles, 2) a.Equal(*seenFiles["folder1/file1.txt"].Metadata["key1"], "val1") - a.Equal(*seenFiles["folder1/file2.txt"].Metadata["key1"], "val1") + a.Equal(*seenFiles["folder1/"].Metadata["key1"], "val1") } // set two bodies where the first blobprefix becomes prefix for second call -func SetHierachicalBodyRecursive(accounturl string, containername string, folderflag bool, secondcall bool) string { +func SetHierarchicalBodyRecursive(accounturl string, containername string, folderflag bool, secondcall bool) string { metadatatest := map[string]*string{ "key1": to.Ptr("val1"), } @@ -925,7 +898,7 @@ func SetHierachicalBodyRecursive(accounturl string, containername string, folder } } -// this test calls Traverse and calls a Parallel List with two responses with the second prefix set as the first blobprefix +// this test calls Traverse and calls a Parallel List with a folder that is skipped and two responses with the second prefix set as the first blobprefix func TestTraverseParallelListWithMarkerRecursive(t *testing.T) { a := assert.New(t) //create mock server @@ -946,14 +919,13 @@ func TestTraverseParallelListWithMarkerRecursive(t *testing.T) { //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyRecursive(rawURL, cName, true, false)))) + //use marshalling to append original body + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBodyRecursive(rawURL, cName, true, false)))) //second body uses first prefix - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyRecursive(rawURL, cName, false, true)))) + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBodyRecursive(rawURL, cName, false, true)))) //create new blob traverser - //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized - //made recursive true + //changed includeversion to be false to make parallelListing true, changed includeDirectoryStubs to false so that hdi_isfolder is recognized, and made recursive true for second call to occur blobTraverser := newBlobTraverser(blobURL, client, ctx, true, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -969,15 +941,14 @@ func TestTraverseParallelListWithMarkerRecursive(t *testing.T) { a.Equal(*seenFiles["folder1/folder2/file2.txt"].Metadata["key1"], "val1") } -// set two bodies where the first blobprefix becomes prefix for second call -func SetHierachicalBodyBlobTags(accounturl string, containername string) string { +// set tags for hierarchical mock response +func SetHierarchicalBodyBlobTags(accounturl string, containername string) string { metadatatest := map[string]*string{ "key1": to.Ptr("val1"), } blobProp := container.BlobProperties{ContentLength: to.Ptr(int64(2))} blobtags := container.BlobTags{BlobTagSet: []*container.BlobTag{to.Ptr(container.BlobTag{Key: to.Ptr("tagval"), Value: to.Ptr("keyval")})}} blobitem := []*container.BlobItem{to.Ptr(container.BlobItem{Name: to.Ptr("folder1/file1.txt"), Properties: to.Ptr(blobProp), BlobTags: to.Ptr(blobtags), Metadata: metadatatest}), to.Ptr(container.BlobItem{Name: to.Ptr("folder1/folder2/"), Properties: to.Ptr(blobProp), Metadata: metadatatest})} - //blobprefix := []*container.BlobPrefix{to.Ptr(container.BlobPrefix{Name: to.Ptr("folder1/folder2/")})} segment := container.BlobHierarchyListSegment{BlobItems: blobitem} resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("folder1/")} out, err := xml.Marshal(resp) @@ -989,7 +960,7 @@ func SetHierachicalBodyBlobTags(accounturl string, containername string) string return newbody } -// this test calls Traverse and calls a Parallel List with two responses with the second prefix set as the first blobprefix and sets properties to dir +// this test calls Traverse with a Parallel List with blob tags and metadata stored func TestTraverseParallelListBlobTags(t *testing.T) { a := assert.New(t) //create mock server @@ -1007,14 +978,13 @@ func TestTraverseParallelListBlobTags(t *testing.T) { }}) a.Nil(err) - //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyBlobTags(rawURL, cName)))) + //use marshalling to append hierarchical body with tags + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBodyBlobTags(rawURL, cName)))) //create new blob traverser - //changed includeversion to be false to make parallelListing true and changed includeDirectoryStubs to false*** so that hdi_isfolder is recognized and made recursive true + //changed includeversion to be false to make parallelListing true, changed includeDirectoryStubs to false so that hdi_isfolder is recognized, and made recursive true blobTraverser := newBlobTraverser(blobURL, client, ctx, true, true, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) //test method and validate @@ -1031,7 +1001,7 @@ func TestTraverseParallelListBlobTags(t *testing.T) { a.Equal(seenFiles["folder1/file1.txt"].blobTags["tagval"], "keyval") } -func SetHierachicalBodyNothing(accounturl string, containername string) string { +func SetHierarchicalBodyNoBlob(accounturl string, containername string) string { segment := container.BlobHierarchyListSegment{} resp := container.ListBlobsHierarchySegmentResponse{ContainerName: to.Ptr(containername), Segment: to.Ptr(segment), ServiceEndpoint: to.Ptr(accounturl), Delimiter: to.Ptr("/"), Marker: to.Ptr(""), MaxResults: to.Ptr(int32(10)), Prefix: to.Ptr("")} out, err := xml.Marshal(resp) @@ -1044,7 +1014,7 @@ func SetHierachicalBodyNothing(accounturl string, containername string) string { } // this test calls Traverse and calls a Parallel List with no blob item or blob prefix -func TestTraverseParallelListNothing(t *testing.T) { +func TestTraverseParallelListNoBlob(t *testing.T) { a := assert.New(t) //create mock server srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl()) @@ -1059,11 +1029,10 @@ func TestTraverseParallelListNothing(t *testing.T) { Transport: srv, //passing in mock server }}) a.Nil(err) - //define what mock server should return as response //first case is not a blob srv.AppendResponse(mock_server.WithStatusCode(400)) - //use marshalling to append body - srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierachicalBodyNothing(rawURL, cName)))) + //use marshalling to append body with no blob + srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(SetHierarchicalBodyNoBlob(rawURL, cName)))) //create new blob traverser blobTraverser := newBlobTraverser(blobURL, client, ctx, false, false, func(common.EntityType) {}, true, common.CpkOptions{}, true, false, false, common.EPreservePermissionsOption.None(), false) diff --git a/cmd/zt_traverser_type.go b/cmd/zt_traverser_type.go deleted file mode 100644 index 3d9649794..000000000 --- a/cmd/zt_traverser_type.go +++ /dev/null @@ -1,131 +0,0 @@ -package cmd - -import ( - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "time" -) - -// ListBlobsFlatSegmentResponse - An enumeration of blobs -type ListBlobsFlatSegmentResponse struct { - // REQUIRED - ContainerName *string `xml:"ContainerName,attr"` - - // REQUIRED - Segment *BlobFlatListSegment `xml:"Blobs"` - - // REQUIRED - ServiceEndpoint *string `xml:"ServiceEndpoint,attr"` - Marker *string `xml:"Marker"` - MaxResults *int32 `xml:"MaxResults"` - NextMarker *string `xml:"NextMarker"` - Prefix *string `xml:"Prefix"` -} - -type BlobFlatListSegment struct { - // REQUIRED - BlobItems []*BlobItem `xml:"Blob"` -} - -// BlobItem - An Azure Storage blob -type BlobItem struct { - // REQUIRED - Deleted *bool `xml:"Deleted"` - - // REQUIRED - Name *string `xml:"Name"` - - // REQUIRED; Properties of a blob - Properties *BlobProperties `xml:"Properties"` - - // REQUIRED - Snapshot *string `xml:"Snapshot"` - - // Blob tags - BlobTags *BlobTags `xml:"Tags"` - HasVersionsOnly *bool `xml:"HasVersionsOnly"` - IsCurrentVersion *bool `xml:"IsCurrentVersion"` - - // Dictionary of - Metadata map[string]*string `xml:"Metadata"` - - // Dictionary of - OrMetadata map[string]*string `xml:"OrMetadata"` - VersionID *string `xml:"VersionId"` -} - -// BlobProperties - Properties of a blob -type BlobProperties struct { - // REQUIRED - ETag *azcore.ETag `xml:"Etag"` - - // REQUIRED - LastModified *time.Time `xml:"Last-Modified"` - AccessTier *AccessTier `xml:"AccessTier"` - AccessTierChangeTime *time.Time `xml:"AccessTierChangeTime"` - AccessTierInferred *bool `xml:"AccessTierInferred"` - ArchiveStatus *ArchiveStatus `xml:"ArchiveStatus"` - BlobSequenceNumber *int64 `xml:"x-ms-blob-sequence-number"` - BlobType *BlobType `xml:"BlobType"` - CacheControl *string `xml:"Cache-Control"` - ContentDisposition *string `xml:"Content-Disposition"` - ContentEncoding *string `xml:"Content-Encoding"` - ContentLanguage *string `xml:"Content-Language"` - - // Size in bytes - ContentLength *int64 `xml:"Content-Length"` - ContentMD5 []byte `xml:"Content-MD5"` - ContentType *string `xml:"Content-Type"` - CopyCompletionTime *time.Time `xml:"CopyCompletionTime"` - CopyID *string `xml:"CopyId"` - CopyProgress *string `xml:"CopyProgress"` - CopySource *string `xml:"CopySource"` - CopyStatus *CopyStatusType `xml:"CopyStatus"` - CopyStatusDescription *string `xml:"CopyStatusDescription"` - CreationTime *time.Time `xml:"Creation-Time"` - CustomerProvidedKeySHA256 *string `xml:"CustomerProvidedKeySha256"` - DeletedTime *time.Time `xml:"DeletedTime"` - DestinationSnapshot *string `xml:"DestinationSnapshot"` - - // The name of the encryption scope under which the blob is encrypted. - EncryptionScope *string `xml:"EncryptionScope"` - ExpiresOn *time.Time `xml:"Expiry-Time"` - ImmutabilityPolicyExpiresOn *time.Time `xml:"ImmutabilityPolicyUntilDate"` - ImmutabilityPolicyMode *ImmutabilityPolicyMode `xml:"ImmutabilityPolicyMode"` - IncrementalCopy *bool `xml:"IncrementalCopy"` - IsSealed *bool `xml:"Sealed"` - LastAccessedOn *time.Time `xml:"LastAccessTime"` - LeaseDuration *LeaseDurationType `xml:"LeaseDuration"` - LeaseState *LeaseStateType `xml:"LeaseState"` - LeaseStatus *LeaseStatusType `xml:"LeaseStatus"` - LegalHold *bool `xml:"LegalHold"` - - // If an object is in rehydrate pending state then this header is returned with priority of rehydrate. Valid values are High - // and Standard. - RehydratePriority *RehydratePriority `xml:"RehydratePriority"` - RemainingRetentionDays *int32 `xml:"RemainingRetentionDays"` - ServerEncrypted *bool `xml:"ServerEncrypted"` - TagCount *int32 `xml:"TagCount"` -} - -type BlobTags struct { - // REQUIRED - BlobTagSet []*BlobTag `xml:"TagSet>Tag"` -} - -type BlobTag struct { - // REQUIRED - Key *string `xml:"Key"` - - // REQUIRED - Value *string `xml:"Value"` -} - -type BlobType string -type AccessTier string -type ArchiveStatus string -type CopyStatusType string -type ImmutabilityPolicyMode string -type LeaseDurationType string -type LeaseStateType string -type LeaseStatusType string -type RehydratePriority string