Add support for multi-store import

This change adds support for providing multiple target
stores where image can be imported.

Co-authored-by: Erno Kuvaja <jokke@usr.fi>
Co-authored-by: Abhishek Kekane <akekane@redhat.com>

bp: import-multi-stores
Change-Id: I8730364263f1afd5d11fd56939851bda73a892bb

glanceclient/tests/unit/v2/test_shell_v2.py

@@ -870,6 +870,90 @@ def test_neg_image_create_via_import_no_method_with_file_and_stdin(
                 pass
         mock_utils_exit.assert_called_once_with(expected_msg)
 
+    @mock.patch('glanceclient.common.utils.exit')
+    def test_neg_image_create_via_import_stores_all_stores_specified(
+            self, mock_utils_exit):
+        expected_msg = ('Only one of --store, --stores and --all-stores can '
+                        'be provided')
+        mock_utils_exit.side_effect = self._mock_utils_exit
+        my_args = self.base_args.copy()
+        my_args.update(
+            {'id': 'IMG-01', 'import_method': 'glance-direct',
+             'stores': 'file1,file2', 'os_all_stores': True,
+             'file': 'some.mufile',
+             'disk_format': 'raw',
+             'container_format': 'bare',
+             })
+        args = self._make_args(my_args)
+
+        with mock.patch.object(self.gc.images,
+                               'get_import_info') as mocked_info:
+            mocked_info.return_value = self.import_info_response
+            try:
+                test_shell.do_image_create_via_import(self.gc, args)
+                self.fail("utils.exit should have been called")
+            except SystemExit:
+                pass
+        mock_utils_exit.assert_called_once_with(expected_msg)
+
+    @mock.patch('glanceclient.common.utils.exit')
+    @mock.patch('sys.stdin', autospec=True)
+    def test_neg_image_create_via_import_stores_without_file(
+            self, mock_stdin, mock_utils_exit):
+        expected_msg = ('--stores option should only be provided with --file '
+                        'option or stdin for the glance-direct import method.')
+        mock_utils_exit.side_effect = self._mock_utils_exit
+        mock_stdin.isatty = lambda: True
+        my_args = self.base_args.copy()
+        my_args.update(
+            {'id': 'IMG-01', 'import_method': 'glance-direct',
+             'stores': 'file1,file2',
+             'disk_format': 'raw',
+             'container_format': 'bare',
+             })
+        args = self._make_args(my_args)
+
+        with mock.patch.object(self.gc.images,
+                               'get_import_info') as mocked_info:
+            with mock.patch.object(self.gc.images,
+                                   'get_stores_info') as mocked_stores_info:
+                mocked_stores_info.return_value = self.stores_info_response
+                mocked_info.return_value = self.import_info_response
+                try:
+                    test_shell.do_image_create_via_import(self.gc, args)
+                    self.fail("utils.exit should have been called")
+                except SystemExit:
+                    pass
+        mock_utils_exit.assert_called_once_with(expected_msg)
+
+    @mock.patch('glanceclient.common.utils.exit')
+    @mock.patch('sys.stdin', autospec=True)
+    def test_neg_image_create_via_import_all_stores_without_file(
+            self, mock_stdin, mock_utils_exit):
+        expected_msg = ('--all-stores option should only be provided with '
+                        '--file option or stdin for the glance-direct import '
+                        'method.')
+        mock_utils_exit.side_effect = self._mock_utils_exit
+        mock_stdin.isatty = lambda: True
+        my_args = self.base_args.copy()
+        my_args.update(
+            {'id': 'IMG-01', 'import_method': 'glance-direct',
+             'os_all_stores': True,
+             'disk_format': 'raw',
+             'container_format': 'bare',
+             })
+        args = self._make_args(my_args)
+
+        with mock.patch.object(self.gc.images,
+                               'get_import_info') as mocked_info:
+            mocked_info.return_value = self.import_info_response
+            try:
+                test_shell.do_image_create_via_import(self.gc, args)
+                self.fail("utils.exit should have been called")
+            except SystemExit:
+                pass
+        mock_utils_exit.assert_called_once_with(expected_msg)
+
     @mock.patch('glanceclient.common.utils.exit')
     @mock.patch('os.access')
     @mock.patch('sys.stdin', autospec=True)
@@ -1083,6 +1167,60 @@ def test_neg_image_create_via_import_web_download_no_uri(
                 pass
         mock_utils_exit.assert_called_once_with(expected_msg)
 
+    @mock.patch('glanceclient.common.utils.exit')
+    def test_neg_image_create_via_import_stores_without_uri(
+            self, mock_utils_exit):
+        expected_msg = ('--stores option should only be provided with --uri '
+                        'option for the web-download import method.')
+        mock_utils_exit.side_effect = self._mock_utils_exit
+        my_args = self.base_args.copy()
+        my_args.update(
+            {'id': 'IMG-01', 'import_method': 'web-download',
+             'stores': 'file1,file2',
+             'disk_format': 'raw',
+             'container_format': 'bare',
+             })
+        args = self._make_args(my_args)
+
+        with mock.patch.object(self.gc.images,
+                               'get_import_info') as mocked_info:
+            with mock.patch.object(self.gc.images,
+                                   'get_stores_info') as mocked_stores_info:
+                mocked_stores_info.return_value = self.stores_info_response
+                mocked_info.return_value = self.import_info_response
+                try:
+                    test_shell.do_image_create_via_import(self.gc, args)
+                    self.fail("utils.exit should have been called")
+                except SystemExit:
+                    pass
+        mock_utils_exit.assert_called_once_with(expected_msg)
+
+    @mock.patch('glanceclient.common.utils.exit')
+    def test_neg_image_create_via_import_all_stores_without_uri(
+            self, mock_utils_exit):
+        expected_msg = ('--all-stores option should only be provided with '
+                        '--uri option for the web-download import '
+                        'method.')
+        mock_utils_exit.side_effect = self._mock_utils_exit
+        my_args = self.base_args.copy()
+        my_args.update(
+            {'id': 'IMG-01', 'import_method': 'web-download',
+             'os_all_stores': True,
+             'disk_format': 'raw',
+             'container_format': 'bare',
+             })
+        args = self._make_args(my_args)
+
+        with mock.patch.object(self.gc.images,
+                               'get_import_info') as mocked_info:
+            mocked_info.return_value = self.import_info_response
+            try:
+                test_shell.do_image_create_via_import(self.gc, args)
+                self.fail("utils.exit should have been called")
+            except SystemExit:
+                pass
+        mock_utils_exit.assert_called_once_with(expected_msg)
+
     @mock.patch('glanceclient.common.utils.exit')
     @mock.patch('sys.stdin', autospec=True)
     def test_neg_image_create_via_import_web_download_no_uri_with_file(
@@ -1785,7 +1923,8 @@ def test_image_import_glance_direct(self):
                     mock_import.return_value = None
                     test_shell.do_image_import(self.gc, args)
                     mock_import.assert_called_once_with(
-                        'IMG-01', 'glance-direct', None, backend=None)
+                        'IMG-01', 'glance-direct', None, backend=None,
+                        all_stores=None, allow_failure=True, stores=None)
 
     def test_image_import_web_download(self):
         args = self._make_args(
@@ -1803,7 +1942,9 @@ def test_image_import_web_download(self):
                     test_shell.do_image_import(self.gc, args)
                     mock_import.assert_called_once_with(
                         'IMG-01', 'web-download',
-                        'http://example.com/image.qcow', backend=None)
+                        'http://example.com/image.qcow',
+                        all_stores=None, allow_failure=True,
+                        backend=None, stores=None)
 
     @mock.patch('glanceclient.common.utils.print_image')
     def test_image_import_no_print_image(self, mocked_utils_print_image):
@@ -1821,9 +1962,32 @@ def test_image_import_no_print_image(self, mocked_utils_print_image):
                     mock_import.return_value = None
                     test_shell.do_image_import(self.gc, args)
                     mock_import.assert_called_once_with(
-                        'IMG-02', 'glance-direct', None, backend=None)
+                        'IMG-02', 'glance-direct', None, stores=None,
+                        all_stores=None, allow_failure=True, backend=None)
                     mocked_utils_print_image.assert_not_called()
 
+    @mock.patch('glanceclient.common.utils.print_image')
+    @mock.patch('glanceclient.v2.shell._validate_backend')
+    def test_image_import_multiple_stores(self, mocked_utils_print_image,
+                                          msvb):
+        args = self._make_args(
+            {'id': 'IMG-02', 'uri': None, 'import_method': 'glance-direct',
+                'from_create': False, 'stores': 'site1,site2'})
+        with mock.patch.object(self.gc.images, 'image_import') as mock_import:
+            with mock.patch.object(self.gc.images, 'get') as mocked_get:
+                with mock.patch.object(self.gc.images,
+                                       'get_import_info') as mocked_info:
+                    mocked_get.return_value = {'status': 'uploading',
+                                               'container_format': 'bare',
+                                               'disk_format': 'raw'}
+                    mocked_info.return_value = self.import_info_response
+                    mock_import.return_value = None
+                    test_shell.do_image_import(self.gc, args)
+                    mock_import.assert_called_once_with(
+                        'IMG-02', 'glance-direct', None, all_stores=None,
+                        allow_failure=True, stores=['site1', 'site2'],
+                        backend=None)
+
     def test_image_download(self):
         args = self._make_args(
             {'id': 'IMG-01', 'file': 'test', 'progress': True,

glanceclient/v2/images.py

@@ -318,13 +318,22 @@ def stage(self, image_id, image_data, image_size=None):
 
     @utils.add_req_id_to_object()
     def image_import(self, image_id, method='glance-direct', uri=None,
-                     backend=None):
+                     backend=None, stores=None, allow_failure=True,
+                     all_stores=None):
         """Import Image via method."""
         headers = {}
         url = '/v2/images/%s/import' % image_id
         data = {'method': {'name': method}}
+        if stores:
+            data['stores'] = stores
+            if allow_failure:
+                data['all_stores_must_succeed'] = 'false'
         if backend is not None:
             headers['x-image-meta-store'] = backend
+        if all_stores:
+            data['all_stores'] = 'true'
+            if allow_failure:
+                data['all_stores_must_succeed'] = 'false'
 
         if uri:
             if method == 'web-download':

glanceclient/v2/shell.py

@@ -150,6 +150,28 @@ def do_image_create(gc, args):
 @utils.arg('--store', metavar='<STORE>',
            default=utils.env('OS_IMAGE_STORE', default=None),
            help='Backend store to upload image to.')
+@utils.arg('--stores', metavar='<STORES>',
+           default=utils.env('OS_IMAGE_STORES', default=None),
+           help=_('Stores to upload image to if multi-stores import '
+                  'available. Comma separated list. Available stores can be '
+                  'listed with "stores-info" call.'))
+@utils.arg('--all-stores', type=strutils.bool_from_string,
+           metavar='[True|False]',
+           default=None,
+           dest='os_all_stores',
+           help=_('"all-stores" can be ued instead of "stores"-list to '
+                  'indicate that image should be imported into all available '
+                  'stores.'))
+@utils.arg('--allow-failure', type=strutils.bool_from_string,
+           metavar='[True|False]',
+           dest='os_allow_failure',
+           default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True),
+           help=_('Indicator if all stores listed (or available) must '
+                  'succeed. "True" by default meaning that we allow some '
+                  'stores to fail and the status can be monitored from the '
+                  'image metadata. If this is set to "False" the import will '
+                  'be reverted should any of the uploads fail. Only usable '
+                  'with "stores" or "all-stores".'))
 @utils.on_data_require_fields(DATA_FIELDS)
 def do_image_create_via_import(gc, args):
     """EXPERIMENTAL: Create a new image via image import.
@@ -198,9 +220,21 @@ def do_image_create_via_import(gc, args):
 
     # determine if backend is valid
     backend = None
-    if args.store:
+    stores = getattr(args, "stores", None)
+    all_stores = getattr(args, "os_all_stores", None)
+
+    if (args.store and (stores or all_stores)) or (stores and all_stores):
+        utils.exit("Only one of --store, --stores and --all-stores can be "
+                   "provided")
+    elif args.store:
         backend = args.store
+        # determine if backend is valid
         _validate_backend(backend, gc)
+    elif stores:
+        stores = str(stores).split(',')
+        for store in stores:
+            # determine if backend is valid
+            _validate_backend(store, gc)
 
     # make sure we have all and only correct inputs for the requested method
     if args.import_method is None:
@@ -211,6 +245,14 @@ def do_image_create_via_import(gc, args):
         if backend and not (file_name or using_stdin):
             utils.exit("--store option should only be provided with --file "
                        "option or stdin for the glance-direct import method.")
+        if stores and not (file_name or using_stdin):
+            utils.exit("--stores option should only be provided with --file "
+                       "option or stdin for the glance-direct import method.")
+        if all_stores and not (file_name or using_stdin):
+            utils.exit("--all-stores option should only be provided with "
+                       "--file option or stdin for the glance-direct import "
+                       "method.")
+
         if args.uri:
             utils.exit("You cannot specify a --uri with the glance-direct "
                        "import method.")
@@ -227,6 +269,12 @@ def do_image_create_via_import(gc, args):
         if backend and not args.uri:
             utils.exit("--store option should only be provided with --uri "
                        "option for the web-download import method.")
+        if stores and not args.uri:
+            utils.exit("--stores option should only be provided with --uri "
+                       "option for the web-download import method.")
+        if all_stores and not args.uri:
+            utils.exit("--all-stores option should only be provided with "
+                       "--uri option for the web-download import method.")
         if not args.uri:
             utils.exit("URI is required for web-download import method. "
                        "Please use '--uri <uri>'.")
@@ -246,6 +294,7 @@ def do_image_create_via_import(gc, args):
                 args.size = None
                 do_image_stage(gc, args)
             args.from_create = True
+            args.stores = stores
             do_image_import(gc, args)
         image = gc.images.get(args.id)
     finally:
@@ -617,19 +666,56 @@ def do_image_stage(gc, args):
 @utils.arg('--store', metavar='<STORE>',
            default=utils.env('OS_IMAGE_STORE', default=None),
            help='Backend store to upload image to.')
+@utils.arg('--stores', metavar='<STORES>',
+           default=utils.env('OS_IMAGE_STORES', default=None),
+           help='Stores to upload image to if multi-stores import available.')
+@utils.arg('--all-stores', type=strutils.bool_from_string,
+           metavar='[True|False]',
+           default=None,
+           dest='os_all_stores',
+           help=_('"all-stores" can be ued instead of "stores"-list to '
+                  'indicate that image should be imported all available '
+                  'stores.'))
+@utils.arg('--allow-failure', type=strutils.bool_from_string,
+           metavar='[True|False]',
+           dest='os_allow_failure',
+           default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True),
+           help=_('Indicator if all stores listed (or available) must '
+                  'succeed. "True" by default meaning that we allow some '
+                  'stores to fail and the status can be monitored from the '
+                  'image metadata. If this is set to "False" the import will '
+                  'be reverted should any of the uploads fail. Only usable '
+                  'with "stores" or "all-stores".'))
 def do_image_import(gc, args):
     """Initiate the image import taskflow."""
-    backend = None
-    if args.store:
-        backend = args.store
+    backend = getattr(args, "store", None)
+    stores = getattr(args, "stores", None)
+    all_stores = getattr(args, "os_all_stores", None)
+    allow_failure = getattr(args, "os_allow_failure", True)
+
+    if not getattr(args, 'from_create', False):
+        if (args.store and (stores or all_stores)) or (stores and all_stores):
+            utils.exit("Only one of --store, --stores and --all-stores can be "
+                       "provided")
+        elif args.store:
+            backend = args.store
+            # determine if backend is valid
+            _validate_backend(backend, gc)
+        elif stores:
+            stores = str(stores).split(',')
+
         # determine if backend is valid
-        _validate_backend(backend, gc)
+        if stores:
+            for store in stores:
+                _validate_backend(store, gc)
 
     if getattr(args, 'from_create', False):
         # this command is being called "internally" so we can skip
         # validation -- just do the import and get out of here
         gc.images.image_import(args.id, args.import_method, args.uri,
-                               backend=backend)
+                               backend=backend,
+                               stores=stores, all_stores=all_stores,
+                               allow_failure=allow_failure)
         return
 
     # do input validation
@@ -669,7 +755,9 @@ def do_image_import(gc, args):
 
     # finally, do the import
     gc.images.image_import(args.id, args.import_method, args.uri,
-                           backend=backend)
+                           backend=backend,
+                           stores=stores, all_stores=all_stores,
+                           allow_failure=allow_failure)
 
     image = gc.images.get(args.id)
     utils.print_image(image)

releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for multi-store import where user can import
+    image into multiple backend stores with single command.