From c23d86738fcfd55301471e8da2512642fd66eba3 Mon Sep 17 00:00:00 2001 From: Erno Kuvaja Date: Wed, 18 Dec 2019 10:30:54 +0000 Subject: [PATCH] Add support for multi-store import This change adds support for providing multiple target stores where image can be imported. Co-authored-by: Erno Kuvaja Co-authored-by: Abhishek Kekane bp: import-multi-stores Change-Id: I8730364263f1afd5d11fd56939851bda73a892bb --- glanceclient/tests/unit/v2/test_shell_v2.py | 170 +++++++++++++++++- glanceclient/v2/images.py | 11 +- glanceclient/v2/shell.py | 102 ++++++++++- .../multi-store-import-45d05a6193ef2c04.yaml | 5 + 4 files changed, 277 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml diff --git a/glanceclient/tests/unit/v2/test_shell_v2.py b/glanceclient/tests/unit/v2/test_shell_v2.py index eb3750039..994993c14 100644 --- a/glanceclient/tests/unit/v2/test_shell_v2.py +++ b/glanceclient/tests/unit/v2/test_shell_v2.py @@ -870,6 +870,90 @@ def test_neg_image_create_via_import_no_method_with_file_and_stdin( pass mock_utils_exit.assert_called_once_with(expected_msg) + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_all_stores_specified( + self, mock_utils_exit): + expected_msg = ('Only one of --store, --stores and --all-stores can ' + 'be provided') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', 'os_all_stores': True, + 'file': 'some.mufile', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --file ' + 'option or stdin for the glance-direct import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_all_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--file option or stdin for the glance-direct import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) @@ -1083,6 +1167,60 @@ def test_neg_image_create_via_import_web_download_no_uri( pass mock_utils_exit.assert_called_once_with(expected_msg) + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --uri ' + 'option for the web-download import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_all_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--uri option for the web-download import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + @mock.patch('glanceclient.common.utils.exit') @mock.patch('sys.stdin', autospec=True) def test_neg_image_create_via_import_web_download_no_uri_with_file( @@ -1785,7 +1923,8 @@ def test_image_import_glance_direct(self): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-01', 'glance-direct', None, backend=None) + 'IMG-01', 'glance-direct', None, backend=None, + all_stores=None, allow_failure=True, stores=None) def test_image_import_web_download(self): args = self._make_args( @@ -1803,7 +1942,9 @@ def test_image_import_web_download(self): test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( 'IMG-01', 'web-download', - 'http://example.com/image.qcow', backend=None) + 'http://example.com/image.qcow', + all_stores=None, allow_failure=True, + backend=None, stores=None) @mock.patch('glanceclient.common.utils.print_image') def test_image_import_no_print_image(self, mocked_utils_print_image): @@ -1821,9 +1962,32 @@ def test_image_import_no_print_image(self, mocked_utils_print_image): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-02', 'glance-direct', None, backend=None) + 'IMG-02', 'glance-direct', None, stores=None, + all_stores=None, allow_failure=True, backend=None) mocked_utils_print_image.assert_not_called() + @mock.patch('glanceclient.common.utils.print_image') + @mock.patch('glanceclient.v2.shell._validate_backend') + def test_image_import_multiple_stores(self, mocked_utils_print_image, + msvb): + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'glance-direct', + 'from_create': False, 'stores': 'site1,site2'}) + with mock.patch.object(self.gc.images, 'image_import') as mock_import: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_get.return_value = {'status': 'uploading', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + mock_import.return_value = None + test_shell.do_image_import(self.gc, args) + mock_import.assert_called_once_with( + 'IMG-02', 'glance-direct', None, all_stores=None, + allow_failure=True, stores=['site1', 'site2'], + backend=None) + def test_image_download(self): args = self._make_args( {'id': 'IMG-01', 'file': 'test', 'progress': True, diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index 5252ee3f7..69163fe8e 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -318,13 +318,22 @@ def stage(self, image_id, image_data, image_size=None): @utils.add_req_id_to_object() def image_import(self, image_id, method='glance-direct', uri=None, - backend=None): + backend=None, stores=None, allow_failure=True, + all_stores=None): """Import Image via method.""" headers = {} url = '/v2/images/%s/import' % image_id data = {'method': {'name': method}} + if stores: + data['stores'] = stores + if allow_failure: + data['all_stores_must_succeed'] = 'false' if backend is not None: headers['x-image-meta-store'] = backend + if all_stores: + data['all_stores'] = 'true' + if allow_failure: + data['all_stores_must_succeed'] = 'false' if uri: if method == 'web-download': diff --git a/glanceclient/v2/shell.py b/glanceclient/v2/shell.py index 8b2c13bcf..b3ac56f6e 100644 --- a/glanceclient/v2/shell.py +++ b/glanceclient/v2/shell.py @@ -150,6 +150,28 @@ def do_image_create(gc, args): @utils.arg('--store', metavar='', default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='', + default=utils.env('OS_IMAGE_STORES', default=None), + help=_('Stores to upload image to if multi-stores import ' + 'available. Comma separated list. Available stores can be ' + 'listed with "stores-info" call.')) +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported into all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) @utils.on_data_require_fields(DATA_FIELDS) def do_image_create_via_import(gc, args): """EXPERIMENTAL: Create a new image via image import. @@ -198,9 +220,21 @@ def do_image_create_via_import(gc, args): # determine if backend is valid backend = None - if args.store: + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: backend = args.store + # determine if backend is valid _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + for store in stores: + # determine if backend is valid + _validate_backend(store, gc) # make sure we have all and only correct inputs for the requested method if args.import_method is None: @@ -211,6 +245,14 @@ def do_image_create_via_import(gc, args): if backend and not (file_name or using_stdin): utils.exit("--store option should only be provided with --file " "option or stdin for the glance-direct import method.") + if stores and not (file_name or using_stdin): + utils.exit("--stores option should only be provided with --file " + "option or stdin for the glance-direct import method.") + if all_stores and not (file_name or using_stdin): + utils.exit("--all-stores option should only be provided with " + "--file option or stdin for the glance-direct import " + "method.") + if args.uri: utils.exit("You cannot specify a --uri with the glance-direct " "import method.") @@ -227,6 +269,12 @@ def do_image_create_via_import(gc, args): if backend and not args.uri: utils.exit("--store option should only be provided with --uri " "option for the web-download import method.") + if stores and not args.uri: + utils.exit("--stores option should only be provided with --uri " + "option for the web-download import method.") + if all_stores and not args.uri: + utils.exit("--all-stores option should only be provided with " + "--uri option for the web-download import method.") if not args.uri: utils.exit("URI is required for web-download import method. " "Please use '--uri '.") @@ -246,6 +294,7 @@ def do_image_create_via_import(gc, args): args.size = None do_image_stage(gc, args) args.from_create = True + args.stores = stores do_image_import(gc, args) image = gc.images.get(args.id) finally: @@ -617,19 +666,56 @@ def do_image_stage(gc, args): @utils.arg('--store', metavar='', default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='', + default=utils.env('OS_IMAGE_STORES', default=None), + help='Stores to upload image to if multi-stores import available.') +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) def do_image_import(gc, args): """Initiate the image import taskflow.""" - backend = None - if args.store: - backend = args.store + backend = getattr(args, "store", None) + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + allow_failure = getattr(args, "os_allow_failure", True) + + if not getattr(args, 'from_create', False): + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: + backend = args.store + # determine if backend is valid + _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + # determine if backend is valid - _validate_backend(backend, gc) + if stores: + for store in stores: + _validate_backend(store, gc) if getattr(args, 'from_create', False): # this command is being called "internally" so we can skip # validation -- just do the import and get out of here gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) return # do input validation @@ -669,7 +755,9 @@ def do_image_import(gc, args): # finally, do the import gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) image = gc.images.get(args.id) utils.print_image(image) diff --git a/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml b/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml new file mode 100644 index 000000000..8b483e375 --- /dev/null +++ b/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for multi-store import where user can import + image into multiple backend stores with single command.