Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Knack conversion #19

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 43 additions & 41 deletions src/image-copy/azext_imagecopy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,44 +3,46 @@
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

from azure.cli.core.help_files import helps
from azure.cli.core.sdk.util import ParametersContext

helps['image copy'] = """
type: command
short-summary: Copy a managed image (or vm) to other regions
long-summary: >
Allows to copy a managed image (or vm) to other regions.
Keep in mind that it requires the source disk to be available.
examples:
- name: Copy an image to several regions and cleanup at the end.
text: >
az image copy --source-resource-group mySources-rg --source-object-name myImage \\
--target-location uksouth northeurope --target-resource-group "images-repo-rg" --cleanup
- name: Use an already generalized vm to create images in other regions.
text: >
az image copy --source-resource-group mySources-rg --source-object-name myVm \\
--source-type vm --target-location uksouth northeurope --target-resource-group "images-repo-rg"
"""


def load_params(_):
with ParametersContext('image copy') as c:
c.register('source_resource_group_name', '--source-resource-group',
help='Name of the resource group of the source resource')
c.register('source_object_name', '--source-object-name',
help='The name of the image or vm resource')
c.register('target_location', '--target-location', nargs='+',
help='Space separated location list to create the image in (e.g. westeurope etc.)')
c.register('source_type', '--source-type', default='image', choices=['image', 'vm'], help='image or vm')
c.register('target_resource_group_name', '--target-resource-group',
help='Name of the resource group to create images in')
c.register('parallel_degree', '--parallel-degree', type=int, default=-1,
help='Number of parallel copy operations')
c.register('cleanup', '--cleanup', action='store_true', default=False,
help='Include this switch to delete temporary resources upon completion')


def load_commands():
from azure.cli.core.commands import cli_command
cli_command(__name__, 'image copy', 'azext_imagecopy.custom#imagecopy')
from azure.cli.core import AzCommandsLoader

import azext_imagecopy._help # pylint: disable=unused-import


class ImageCopyCommandsLoader(AzCommandsLoader):

def __init__(self, cli_ctx=None):
from azure.cli.core.commands import CliCommandType
imagecopy_custom = CliCommandType(
operations_tmpl='azext_imagecopy.custom#{}')
super(ImageCopyCommandsLoader, self).__init__(cli_ctx=cli_ctx,
custom_command_type=imagecopy_custom)

def load_command_table(self, args):
from azure.cli.core.commands import CliCommandType
imagecopy_custom = CliCommandType(
operations_tmpl='azext_imagecopy.custom#{}')

with self.command_group('image') as g:
g.custom_command('copy', 'imagecopy')

return self.command_table

def load_arguments(self, command):
with self.argument_context('image copy') as c:
c.argument('source_resource_group_name', options_list=['--source-resource-group'],
help='Name of the resource group of the source resource')
c.argument('source_object_name', options_list=['--source-object-name'],
help='The name of the image or vm resource')
c.argument('target_location', options_list=['--target-location'], nargs='+',
help='Space separated location list to create the image in (e.g. westeurope etc.)')
c.argument('source_type', options_list=[
'--source-type'], default='image', choices=['image', 'vm'], help='image or vm')
c.argument('target_resource_group_name', options_list=['--target-resource-group'],
help='Name of the resource group to create images in')
c.argument('parallel_degree', options_list=['--parallel-degree'], type=int, default=-1,
help='Number of parallel copy operations')
c.argument('cleanup', options_list=['--cleanup'], action='store_true', default=False,
help='Include this switch to delete temporary resources upon completion')


COMMAND_LOADER_CLS = ImageCopyCommandsLoader
24 changes: 24 additions & 0 deletions src/image-copy/azext_imagecopy/_help.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

from knack.help_files import helps


helps['image copy'] = """
type: command
short-summary: Copy a managed image (or vm) to other regions
long-summary: >
Allows to copy a managed image (or vm) to other regions.
Keep in mind that it requires the source disk to be available.
examples:
- name: Copy an image to several regions and cleanup at the end.
text: >
az image copy --source-resource-group mySources-rg --source-object-name myImage \\
--target-location uksouth northeurope --target-resource-group "images-repo-rg" --cleanup
- name: Use an already generalized vm to create images in other regions.
text: >
az image copy --source-resource-group mySources-rg --source-object-name myVm \\
--source-type vm --target-location uksouth northeurope --target-resource-group "images-repo-rg"
"""
1 change: 1 addition & 0 deletions src/image-copy/azext_imagecopy/azext_metadata.json
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
{
"azext.minCliVersion": "2.0.23"
}
6 changes: 3 additions & 3 deletions src/image-copy/azext_imagecopy/cli_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import json

from subprocess import check_output, STDOUT, CalledProcessError
from azure.cli.core.util import CLIError
import azure.cli.core.azlogging as azlogging
from knack.util import CLIError

logger = azlogging.get_az_logger(__name__)
from knack.log import get_logger
logger = get_logger(__name__)


def run_cli_command(cmd, return_as_json=False):
Expand Down
149 changes: 78 additions & 71 deletions src/image-copy/azext_imagecopy/create_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@
import datetime
import time
from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command
from azure.cli.core.application import APPLICATION
from azure.cli.core.util import CLIError
import azure.cli.core.azlogging as azlogging
from knack.util import CLIError

logger = azlogging.get_az_logger(__name__)
from knack.log import get_logger
logger = get_logger(__name__)

PROGRESS_LINE_LENGTH = 40

Expand All @@ -23,145 +22,153 @@ def create_target_image(location, transient_resource_group_name, source_type, so

subscription_id = get_subscription_id()

subscription_hash = hashlib.sha1(subscription_id.encode("UTF-8")).hexdigest()
subscription_hash = hashlib.sha1(
subscription_id.encode("UTF-8")).hexdigest()
unique_subscription_string = subscription_hash[:7]

# create the target storage account
logger.warn("{0} - Creating target storage account (can be slow sometimes)".format(location))
logger.warn(
"%s - Creating target storage account (can be slow sometimes)", location)
target_storage_account_name = location + unique_subscription_string
cmd = prepare_cli_command(['storage', 'account', 'create',
'--name', target_storage_account_name,
'--resource-group', transient_resource_group_name,
'--location', location,
'--sku', 'Standard_LRS'])
cli_cmd = prepare_cli_command(['storage', 'account', 'create',
'--name', target_storage_account_name,
'--resource-group', transient_resource_group_name,
'--location', location,
'--sku', 'Standard_LRS'])

json_output = run_cli_command(cmd, return_as_json=True)
json_output = run_cli_command(cli_cmd, return_as_json=True)
target_blob_endpoint = json_output['primaryEndpoints']['blob']

# Setup the target storage account
cmd = prepare_cli_command(['storage', 'account', 'keys', 'list',
'--account-name', target_storage_account_name,
'--resource-group', transient_resource_group_name])
cli_cmd = prepare_cli_command(['storage', 'account', 'keys', 'list',
'--account-name', target_storage_account_name,
'--resource-group', transient_resource_group_name])

json_output = run_cli_command(cmd, return_as_json=True)
json_output = run_cli_command(cli_cmd, return_as_json=True)

target_storage_account_key = json_output[0]['value']
logger.debug(target_storage_account_key)

expiry_format = "%Y-%m-%dT%H:%MZ"
expiry = datetime.datetime.utcnow() + datetime.timedelta(hours=1)

cmd = prepare_cli_command(['storage', 'account', 'generate-sas',
'--account-name', target_storage_account_name,
'--account-key', target_storage_account_key,
'--expiry', expiry.strftime(expiry_format),
'--permissions', 'aclrpuw', '--resource-types',
'sco', '--services', 'b', '--https-only'],
output_as_json=False)
cli_cmd = prepare_cli_command(['storage', 'account', 'generate-sas',
'--account-name', target_storage_account_name,
'--account-key', target_storage_account_key,
'--expiry', expiry.strftime(expiry_format),
'--permissions', 'aclrpuw', '--resource-types',
'sco', '--services', 'b', '--https-only'],
output_as_json=False)

sas_token = run_cli_command(cmd)
sas_token = run_cli_command(cli_cmd)
sas_token = sas_token.rstrip("\n\r") # STRANGE
logger.debug("sas token: " + sas_token)
logger.debug("sas token: %s", sas_token)

# create a container in the target blob storage account
logger.warn("{0} - Creating container in the target storage account".format(location))
logger.warn(
"%s - Creating container in the target storage account", location)
target_container_name = 'snapshots'
cmd = prepare_cli_command(['storage', 'container', 'create',
'--name', target_container_name,
'--account-name', target_storage_account_name])
cli_cmd = prepare_cli_command(['storage', 'container', 'create',
'--name', target_container_name,
'--account-name', target_storage_account_name])

run_cli_command(cmd)
run_cli_command(cli_cmd)

# Copy the snapshot to the target region using the SAS URL
blob_name = source_os_disk_snapshot_name + '.vhd'
logger.warn("{0} - Copying blob to target storage account".format(location))
cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start',
'--source-uri', source_os_disk_snapshot_url,
'--destination-blob', blob_name,
'--destination-container', target_container_name,
'--account-name', target_storage_account_name,
'--sas-token', sas_token])
logger.warn(
"%s - Copying blob to target storage account", location)
cli_cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start',
'--source-uri', source_os_disk_snapshot_url,
'--destination-blob', blob_name,
'--destination-container', target_container_name,
'--account-name', target_storage_account_name,
'--sas-token', sas_token])

run_cli_command(cmd)
run_cli_command(cli_cmd)

# Wait for the copy to complete
start_datetime = datetime.datetime.now()
wait_for_blob_copy_operation(blob_name, target_container_name,
target_storage_account_name, azure_pool_frequency, location)
msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now() - start_datetime).ljust(PROGRESS_LINE_LENGTH)
msg = "{0} - Copy time: {1}".format(
location, datetime.datetime.now() - start_datetime).ljust(PROGRESS_LINE_LENGTH)
logger.warn(msg)

# Create the snapshot in the target region from the copied blob
logger.warn("{0} - Creating snapshot in target region from the copied blob".format(location))
target_blob_path = target_blob_endpoint + target_container_name + '/' + blob_name
logger.warn(
"%s - Creating snapshot in target region from the copied blob", location)
target_blob_path = target_blob_endpoint + \
target_container_name + '/' + blob_name
target_snapshot_name = source_os_disk_snapshot_name + '-' + location
cmd = prepare_cli_command(['snapshot', 'create',
'--resource-group', transient_resource_group_name,
'--name', target_snapshot_name,
'--location', location,
'--source', target_blob_path])
cli_cmd = prepare_cli_command(['snapshot', 'create',
'--resource-group', transient_resource_group_name,
'--name', target_snapshot_name,
'--location', location,
'--source', target_blob_path])

json_output = run_cli_command(cmd, return_as_json=True)
json_output = run_cli_command(cli_cmd, return_as_json=True)
target_snapshot_id = json_output['id']

# Create the final image
logger.warn("{0} - Creating final image".format(location))
logger.warn("%s - Creating final image", location)
target_image_name = source_object_name
if source_type != 'image':
target_image_name += '-image'
target_image_name += '-' + location

cmd = prepare_cli_command(['image', 'create',
'--resource-group', target_resource_group_name,
'--name', target_image_name,
'--location', location,
'--source', target_blob_path,
'--os-type', source_os_type,
'--source', target_snapshot_id])
cli_cmd = prepare_cli_command(['image', 'create',
'--resource-group', target_resource_group_name,
'--name', target_image_name,
'--location', location,
'--source', target_blob_path,
'--os-type', source_os_type,
'--source', target_snapshot_id])

run_cli_command(cmd)
run_cli_command(cli_cmd)


def wait_for_blob_copy_operation(blob_name, target_container_name, target_storage_account_name,
azure_pool_frequency, location):
progress_controller = APPLICATION.get_progress_controller()
copy_status = "pending"
prev_progress = -1
while copy_status == "pending":
cmd = prepare_cli_command(['storage', 'blob', 'show',
'--name', blob_name,
'--container-name', target_container_name,
'--account-name', target_storage_account_name])
cli_cmd = prepare_cli_command(['storage', 'blob', 'show',
'--name', blob_name,
'--container-name', target_container_name,
'--account-name', target_storage_account_name])

json_output = run_cli_command(cmd, return_as_json=True)
json_output = run_cli_command(cli_cmd, return_as_json=True)
copy_status = json_output["properties"]["copy"]["status"]
copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split("/")
current_progress = round(int(copy_progress_1) / int(copy_progress_2), 1)
copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split(
"/")
current_progress = round(
int(copy_progress_1) / int(copy_progress_2), 1)

if current_progress != prev_progress:
msg = "{0} - copy progress: {1}%"\
msg = "{0} - Copy progress: {1}%"\
.format(location, str(current_progress))\
.ljust(PROGRESS_LINE_LENGTH) # need to justify since messages overide each other
progress_controller.add(message=msg)
logger.warn(msg)

prev_progress = current_progress

try:
time.sleep(azure_pool_frequency)
except KeyboardInterrupt:
progress_controller.stop()
return

if copy_status == 'success':
progress_controller.stop()
return
else:
logger.error("The copy operation didn't succeed. Last status: %s", copy_status)
logger.error(
"The copy operation didn't succeed. Last status: %s", copy_status)
raise CLIError('Blob copy failed')


def get_subscription_id():
cmd = prepare_cli_command(['account', 'show'])
json_output = run_cli_command(cmd, return_as_json=True)
cli_cmd = prepare_cli_command(['account', 'show'])
json_output = run_cli_command(cli_cmd, return_as_json=True)
subscription_id = json_output['id']

return subscription_id
Loading