Skip to content

Commit

Permalink
[Storage] Fix #29929: az storage copy: Fix when wildcard * is in …
Browse files Browse the repository at this point in the history 
…`--source-file-path` (#30569)

* fix `az storage copy` with wildcard '*' in source share file path

* fix cases when * is in --source-file-path

* fix cases when * is in --source-blob or --source-container
  • Loading branch information
@calvinhzy
calvinhzy authored Dec 26, 2024
1 parent 6632120 commit 7dd0f54
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 16 deletions.
22 changes: 16 additions & 6 deletions src/azure-cli/azure/cli/command_modules/storage/_validators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2025,18 +2025,28 @@ def get_url_with_sas(cmd, namespace, url=None, container=None, blob=None, share=
client = cf_blob_service(cmd.cli_ctx, kwargs)
if blob is None:
blob = ''
from .operations.blob import create_blob_url
url = create_blob_url(client, container, blob, snapshot=None)
if '*' in container or '*' in blob:
url = client.url
if not url.endswith('/'):
url = url + '/'
url = url + container + '/' + blob
else:
from .operations.blob import create_blob_url
url = create_blob_url(client, container, blob, snapshot=None)
service = 'blob'
elif share:
if hasattr(namespace, 'enable_file_backup_request_intent'):
kwargs.update({'enable_file_backup_request_intent': namespace.enable_file_backup_request_intent})
client = cf_share_service(cmd.cli_ctx, kwargs)
client = client.get_share_client(share)
dir_name, file_name = os.path.split(file_path) if file_path else (None, '')
dir_name = None if dir_name in ('', '.') else dir_name
from .operations.file import create_file_url
url = create_file_url(client, directory_name=dir_name, file_name=file_name)
# if wildcard '*' in file path, skip manually parsing the url
if file_path and '*' in file_path:
url = client.url + '/' + file_path
else:
dir_name, file_name = os.path.split(file_path) if file_path else (None, '')
dir_name = None if dir_name in ('', '.') else dir_name
from .operations.file import create_file_url
url = create_file_url(client, directory_name=dir_name, file_name=file_name)
service = 'file'
elif not any([url, container, share]): # In account level, only blob service is supported
service = 'blob'
Expand Down
77 changes: 67 additions & 10 deletions ...azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_azcopy_scenarios.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,9 +615,8 @@ def test_storage_azcopy_blob_url(self, resource_group, first_account, second_acc
local_file = self.create_temp_file(20480.5)
self.cmd('disk create -n {} -g {} --for-upload --upload-size-bytes 20972032'
.format(diskname, resource_group))
sasURL = self.cmd(
'disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} --query accessSas -o tsv'
.format(diskname, resource_group)).output.strip('\n')
sasURL = self.cmd('disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} -o tsv'
.format(diskname, resource_group)).output.strip('\n')
self.cmd('storage copy -s "{}" -d "{}" --blob-type PageBlob'
.format(local_file, sasURL))

Expand All @@ -632,6 +631,7 @@ def test_storage_azcopy_blob_account(self, resource_group, first_account, second

first_container = self.create_container(first_account_info)
second_container = self.create_container(second_account_info)
third_container = self.create_container(second_account_info)

first_account_url = 'https://{}.blob.core.windows.net'.format(first_account)

Expand Down Expand Up @@ -708,13 +708,20 @@ def test_storage_azcopy_blob_account(self, resource_group, first_account, second
.format(second_container, second_account), checks=JMESPathCheck('length(@)', 11))

# Copy an entire storage account data to another blob account
self.cmd('storage copy --source-account-name {} --destination-account-name {} --recursive --preserve-s2s-access-tier false'
.format(first_account, second_account))
self.cmd('storage copy --source-account-name {} --destination-account-name {} --recursive '
'--preserve-s2s-access-tier false'.format(first_account, second_account))
self.cmd('storage container list --account-name {}'
.format(second_account), checks=JMESPathCheck('length(@)', 2))
.format(second_account), checks=JMESPathCheck('length(@)', 3))
self.cmd('storage blob list -c {} --account-name {}'
.format(first_container, second_account), checks=JMESPathCheck('length(@)', 22))

# Copy an entire directory from blob virtual directory to another blob virtual directory with wildcard
self.cmd('storage copy --source-account-name {} --source-container {} --source-blob {} '
'--destination-account-name {} --destination-container {} --recursive --preserve-s2s-access-tier false'
.format(second_account, second_container, 'apple/*', second_account, third_container))
self.cmd('storage blob list -c {} --account-name {}'
.format(third_container, second_account), checks=JMESPathCheck('length(@)', 10))

@ResourceGroupPreparer()
@StorageAccountPreparer(parameter_name='first_account')
@StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage')
Expand Down Expand Up @@ -881,6 +888,26 @@ def test_storage_azcopy_file_account(self, resource_group, storage_account_info,
self.assertEqual(3, sum(len(d) for r, d, f in os.walk(local_folder)))
self.assertEqual(21, sum(len(f) for r, d, f in os.walk(local_folder)))

# Copy with wildcard *
self.cmd('storage directory create -s {} -n {} --account-name {}'.format(share, 'parent', storage_account))
self.cmd('storage directory create -s {} -n {} --account-name {}'
.format(share, 'parent/source', storage_account))
self.cmd('storage copy --source-local-path "{}" --destination-account-name {} --destination-share {} '
'--destination-file-path {} --recursive'
.format(os.path.join(test_dir, 'butter/file_*'), storage_account, share, 'parent/source'))
self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
'--destination-share {} --destination-file-path {} --recursive -- --as-subdir=false'
.format(storage_account, share, 'parent/source/*', storage_account, share, 'parent/target'))
self.cmd('storage file list -s {} --path {} --account-name {}'.format(
share, 'parent/target', storage_account), checks=JMESPathCheck('length(@)', 10))
self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
'--destination-share {} --destination-file-path {} --recursive -- --as-subdir=false'
.format(storage_account, share, 'parent/*', storage_account, share, 'parentdst'))
self.cmd('storage file list -s {} --path {} --account-name {}'
.format(share, 'parentdst/source', storage_account), checks=JMESPathCheck('length(@)', 10))
self.cmd('storage file list -s {} --path {} --account-name {}'
.format(share, 'parentdst/target', storage_account), checks=JMESPathCheck('length(@)', 10))

@ResourceGroupPreparer()
@StorageAccountPreparer(parameter_name='first_account', allow_shared_key_access=False)
@StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage',
Expand Down Expand Up @@ -972,9 +999,8 @@ def test_storage_azcopy_blob_url_oauth(self, resource_group, first_account, seco
local_file = self.create_temp_file(20480.5)
self.cmd('disk create -n {} -g {} --for-upload --upload-size-bytes 20972032'
.format(diskname, resource_group))
sasURL = self.cmd(
'disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} --query accessSas -o tsv'
.format(diskname, resource_group)).output.strip('\n')
sasURL = self.cmd('disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} -o tsv'
.format(diskname, resource_group)).output.strip('\n')
self.oauth_cmd('storage copy -s "{}" -d "{}" --blob-type PageBlob'.format(
local_file, sasURL))

Expand All @@ -986,8 +1012,10 @@ def test_storage_azcopy_blob_url_oauth(self, resource_group, first_account, seco
def test_storage_azcopy_blob_account_oauth(self, resource_group, first_account, second_account, test_dir):
first_container = self.create_random_name(prefix='container', length=24)
second_container = self.create_random_name(prefix='container', length=24)
third_container = self.create_random_name(prefix='container', length=24)
self.oauth_cmd('storage container create -n {} --account-name {}', first_container, first_account)
self.oauth_cmd('storage container create -n {} --account-name {}', second_container, second_account)
self.oauth_cmd('storage container create -n {} --account-name {}', third_container, second_account)

first_account_url = 'https://{}.blob.core.windows.net'.format(first_account)

Expand Down Expand Up @@ -1072,10 +1100,18 @@ def test_storage_azcopy_blob_account_oauth(self, resource_group, first_account,
'--preserve-s2s-access-tier false'.format(
first_account, second_account))
self.oauth_cmd('storage container list --account-name {}'.format(
second_account), checks=JMESPathCheck('length(@)', 2))
second_account), checks=JMESPathCheck('length(@)', 3))
self.oauth_cmd('storage blob list -c {} --account-name {}'.format(
first_container, second_account), checks=JMESPathCheck('length(@)', 22))

# Copy an entire directory from blob virtual directory to another blob virtual directory with wildcard
self.oauth_cmd('storage copy --source-account-name {} --source-container {} --source-blob {} '
'--destination-account-name {} --destination-container {} --recursive '
'--preserve-s2s-access-tier false'
.format(second_account, second_container, 'apple/*', second_account, third_container))
self.oauth_cmd('storage blob list -c {} --account-name {}'
.format(third_container, second_account), checks=JMESPathCheck('length(@)', 10))

@ResourceGroupPreparer()
@StorageAccountPreparer(parameter_name='first_account', allow_shared_key_access=False)
@StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage',
Expand Down Expand Up @@ -1236,3 +1272,24 @@ def test_storage_azcopy_file_account_oauth(self, resource_group, storage_account
storage_account, share, local_folder))
self.assertEqual(3, sum(len(d) for r, d, f in os.walk(local_folder)))
self.assertEqual(21, sum(len(f) for r, d, f in os.walk(local_folder)))

# Copy with wildcard *
self.file_oauth_cmd('storage directory create -s {} -n {} --account-name {}'
.format(share, 'parent', storage_account))
self.file_oauth_cmd('storage directory create -s {} -n {} --account-name {}'
.format(share, 'parent/source', storage_account))
self.oauth_cmd('storage copy --source-local-path "{}" --destination-account-name {} --destination-share {} '
'--destination-file-path {} --recursive'
.format(os.path.join(test_dir, 'butter/file_*'), storage_account, share, 'parent/source'))
self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
'--destination-share {} --destination-file-path {} --recursive --auth-mode login -- --as-subdir=false'
.format(storage_account, share, 'parent/source/*', storage_account, share, 'parent/target'))
self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
.format(share, 'parent/target', storage_account), checks=JMESPathCheck('length(@)', 10))
self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
'--destination-share {} --destination-file-path {} --recursive --auth-mode login -- --as-subdir=false'
.format(storage_account, share, 'parent/*', storage_account, share, 'parentdst'))
self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
.format(share, 'parentdst/source', storage_account), checks=JMESPathCheck('length(@)', 10))
self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
.format(share, 'parentdst/target', storage_account), checks=JMESPathCheck('length(@)', 10))

0 comments on commit 7dd0f54

Please sign in to comment.