Multihash implementation for Glance

Partially implements blueprint multihash. Requires glance_store 0.26.1 Co-authored-by: Scott McClymont <scott.mcclymont@verizonwireless.com> Co-authored-by: Brian Rosmaita <rosmaita.fossdev@gmail.com> Change-Id: Ib28ea1f6c431db6434dbab2a234018e82d5a6d1a
openstack · Aug 1, 2018 · 0b24dbd · 0b24dbd
1 parent ff77f59
commit 0b24dbd
Show file tree

Hide file tree

Showing 32 changed files with 511 additions and 35 deletions.
diff --git a/api-ref/source/v2/images-images-v2.inc b/api-ref/source/v2/images-images-v2.inc
@@ -202,6 +202,8 @@ Response Parameters
    - min_disk: min_disk
    - min_ram: min_ram
    - name: name
+   - os_hash_algo: os_hash_algo
+   - os_hash_value: os_hash_value
    - owner: owner
    - protected: protected
    - schema: schema-image
@@ -266,6 +268,8 @@ Response Parameters
    - min_disk: min_disk
    - min_ram: min_ram
    - name: name
+   - os_hash_algo: os_hash_algo
+   - os_hash_value: os_hash_value
    - owner: owner
    - protected: protected
    - schema: schema-image
@@ -584,8 +588,10 @@ Response Parameters
    - id: id
    - min_disk: min_disk
    - min_ram: min_ram
-   - owner: owner
    - name: name
+   - owner: owner
+   - os_hash_algo: os_hash_algo
+   - os_hash_value: os_hash_value
    - protected: protected
    - schema: schema-image
    - self: self

diff --git a/api-ref/source/v2/images-parameters.yaml b/api-ref/source/v2/images-parameters.yaml
@@ -484,6 +484,27 @@ next:
   in: body
   required: true
   type: string
+os_hash_algo:
+  description: |
+    The algorithm used to compute a secure hash of the image data for this
+    image.  The result of the computation is displayed as the value of the
+    ``os_hash_value`` property.  The value might be ``null`` (JSON null
+    data type).  The algorithm used is chosen by the cloud operator; it
+    may not be configured by end users.  *(Since Image API v2.7)*
+  in: body
+  required: true
+  type: string
+os_hash_value:
+  description: |
+    The hexdigest of the secure hash of the image data computed using the
+    algorithm whose name is the value of the ``os_hash_algo`` property.
+    The value might be ``null`` (JSON null data type) if data has not
+    yet been associated with this image, or if the image was created using
+    a version of the Image Service API prior to version 2.7.
+    *(Since Image API v2.7)*
+  in: body
+  required: true
+  type: string
 owner:
   description: |
     An identifier for the owner of the image, usually the project (also

diff --git a/api-ref/source/v2/samples/image-create-response.json b/api-ref/source/v2/samples/image-create-response.json
@@ -15,6 +15,8 @@
     "id": "b2173dd3-7ad6-4362-baa6-a68bce3565cb",
     "file": "/v2/images/b2173dd3-7ad6-4362-baa6-a68bce3565cb/file",
     "checksum": null,
+    "os_hash_algo": null,
+    "os_hash_value": null,
     "owner": "bab7d5c60cd041a0a36f7c4b6e1dd978",
     "virtual_size": null,
     "min_ram": 0,

diff --git a/api-ref/source/v2/samples/image-details-deactivate-response.json b/api-ref/source/v2/samples/image-details-deactivate-response.json
@@ -13,6 +13,8 @@
     "id": "1bea47ed-f6a9-463b-b423-14b9cca9ad27",
     "file": "/v2/images/1bea47ed-f6a9-463b-b423-14b9cca9ad27/file",
     "checksum": "64d7c1cd2b6f60c92c14662941cb7913",
+    "os_hash_algo": "sha512",
+    "os_hash_value": "073b4523583784fbe01daff81eba092a262ec37ba6d04dd3f52e4cd5c93eb8258af44881345ecda0e49f3d8cc6d2df6b050ff3e72681d723234aff9d17d0cf09"
     "owner": "5ef70662f8b34079a6eddb8da9d75fe8",
     "size": 13167616,
     "min_ram": 0,

diff --git a/api-ref/source/v2/samples/image-show-response.json b/api-ref/source/v2/samples/image-show-response.json
@@ -13,6 +13,8 @@
     "id": "1bea47ed-f6a9-463b-b423-14b9cca9ad27",
     "file": "/v2/images/1bea47ed-f6a9-463b-b423-14b9cca9ad27/file",
     "checksum": "64d7c1cd2b6f60c92c14662941cb7913",
+    "os_hash_algo": "sha512",
+    "os_hash_value": "073b4523583784fbe01daff81eba092a262ec37ba6d04dd3f52e4cd5c93eb8258af44881345ecda0e49f3d8cc6d2df6b050ff3e72681d723234aff9d17d0cf09"
     "owner": "5ef70662f8b34079a6eddb8da9d75fe8",
     "size": 13167616,
     "min_ram": 0,

diff --git a/api-ref/source/v2/samples/image-update-response.json b/api-ref/source/v2/samples/image-update-response.json
@@ -9,6 +9,8 @@
     "min_ram": 512,
     "name": "Fedora 17",
     "owner": "02a7fb2dd4ef434c8a628c511dcbbeb6",
+    "os_hash_algo": "sha512",
+    "os_hash_value": "ef7d1ed957ffafefb324d50ebc6685ed03d0e64549762ba94a1c44e92270cdbb69d7437dd1e101d00dd41684aaecccad1edc5c2e295e66d4733025b052497844"
     "protected": false,
     "schema": "/v2/schemas/image",
     "self": "/v2/images/2b61ed2b-f800-4da0-99ff-396b742b8646",

diff --git a/api-ref/source/v2/samples/images-list-response.json b/api-ref/source/v2/samples/images-list-response.json
@@ -15,6 +15,8 @@
             "id": "1bea47ed-f6a9-463b-b423-14b9cca9ad27",
             "file": "/v2/images/1bea47ed-f6a9-463b-b423-14b9cca9ad27/file",
             "checksum": "64d7c1cd2b6f60c92c14662941cb7913",
+            "os_hash_algo": "sha512",
+            "os_hash_value": "073b4523583784fbe01daff81eba092a262ec37ba6d04dd3f52e4cd5c93eb8258af44881345ecda0e49f3d8cc6d2df6b050ff3e72681d723234aff9d17d0cf09"
             "owner": "5ef70662f8b34079a6eddb8da9d75fe8",
             "size": 13167616,
             "min_ram": 0,
@@ -36,6 +38,8 @@
             "id": "781b3762-9469-4cec-b58d-3349e5de4e9c",
             "file": "/v2/images/781b3762-9469-4cec-b58d-3349e5de4e9c/file",
             "checksum": "afab0f79bac770d61d24b4d0560b5f70",
+            "os_hash_algo": "sha512",
+            "os_hash_value": "ea3e20140df1cc65f53d4c5b9ee3b38d0d6868f61bbe2230417b0f98cef0e0c7c37f0ebc5c6456fa47f013de48b452617d56c15fdba25e100379bd0e81ee15ec"
             "owner": "5ef70662f8b34079a6eddb8da9d75fe8",
             "size": 476704768,
             "min_ram": 0,

diff --git a/api-ref/source/v2/samples/schemas-image-show-response.json b/api-ref/source/v2/samples/schemas-image-show-response.json
@@ -145,6 +145,24 @@
             "is_base": false,
             "type": "string"
         },
+        "os_hash_algo": {
+            "description": "Algorithm to calculate the os_hash_value",
+            "maxLength": 64,
+            "readOnly": true,
+            "type": [
+                "null",
+                "string"
+            ]
+        },
+        "os_hash_value": {
+            "description": "Hexdigest of the image contents using the algorithm specified by the os_hash_algo",
+            "maxLength": 128,
+            "readOnly": true,
+            "type": [
+                "null",
+                "string"
+            ]
+        },
         "os_version": {
             "description": "Operating system version as specified by the distributor",
             "is_base": false,

diff --git a/api-ref/source/v2/samples/schemas-images-list-response.json b/api-ref/source/v2/samples/schemas-images-list-response.json
@@ -166,6 +166,24 @@
                         "is_base": false,
                         "type": "string"
                     },
+                    "os_hash_algo": {
+                        "description": "Algorithm to calculate the os_hash_value",
+                        "maxLength": 64,
+                        "readOnly": true,
+                        "type": [
+                            "null",
+                            "string"
+                        ]
+                    },
+                    "os_hash_value": {
+                        "description": "Hexdigest of the image contents using the algorithm specified by the os_hash_algo",
+                        "maxLength": 128,
+                        "readOnly": true,
+                        "type": [
+                            "null",
+                            "string"
+                        ]
+                    },
                     "os_version": {
                         "description": "Operating system version as specified by the distributor",
                         "is_base": false,

diff --git a/glance/api/authorization.py b/glance/api/authorization.py
@@ -315,6 +315,8 @@ def __init__(self, base, context):
     min_disk = _immutable_attr('base', 'min_disk')
     min_ram = _immutable_attr('base', 'min_ram')
     protected = _immutable_attr('base', 'protected')
+    os_hash_algo = _immutable_attr('base', 'os_hash_algo')
+    os_hash_value = _immutable_attr('base', 'os_hash_value')
     os_hidden = _immutable_attr('base', 'os_hidden')
     locations = _immutable_attr('base', 'locations', proxy=ImmutableLocations)
     checksum = _immutable_attr('base', 'checksum')

diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py
@@ -446,7 +446,8 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
     _disallowed_properties = ('direct_url', 'self', 'file', 'schema')
     _readonly_properties = ('created_at', 'updated_at', 'status', 'checksum',
                             'size', 'virtual_size', 'direct_url', 'self',
-                            'file', 'schema', 'id')
+                            'file', 'schema', 'id', 'os_hash_algo',
+                            'os_hash_value')
     _reserved_properties = ('location', 'deleted', 'deleted_at')
     _base_properties = ('checksum', 'created_at', 'container_format',
                         'disk_format', 'id', 'min_disk', 'min_ram', 'name',
@@ -884,7 +885,8 @@ def _get_image_locations(image):
             attributes = ['name', 'disk_format', 'container_format',
                           'visibility', 'size', 'virtual_size', 'status',
                           'checksum', 'protected', 'min_ram', 'min_disk',
-                          'owner', 'os_hidden']
+                          'owner', 'os_hidden', 'os_hash_algo',
+                          'os_hash_value']
             for key in attributes:
                 image_view[key] = getattr(image, key)
             image_view['id'] = image.image_id
@@ -1018,6 +1020,19 @@ def get_base_properties():
             'description': _('md5 hash of image contents.'),
             'maxLength': 32,
         },
+        'os_hash_algo': {
+            'type': ['null', 'string'],
+            'readOnly': True,
+            'description': _('Algorithm to calculate the os_hash_value'),
+            'maxLength': 64,
+        },
+        'os_hash_value': {
+            'type': ['null', 'string'],
+            'readOnly': True,
+            'description': _('Hexdigest of the image contents using the '
+                             'algorithm specified by the os_hash_algo'),
+            'maxLength': 128,
+        },
         'owner': {
             'type': ['null', 'string'],
             'description': _('Owner of the image'),

diff --git a/glance/common/config.py b/glance/common/config.py
@@ -191,6 +191,40 @@
 Related options:
     * image_property_quota
 
+""")),
+    cfg.StrOpt('hashing_algorithm',
+               default='sha512',
+               help=_(""""
+Secure hashing algorithm used for computing the 'os_hash_value' property.
+
+This option configures the Glance "multihash", which consists of two
+image properties: the 'os_hash_algo' and the 'os_hash_value'.  The
+'os_hash_algo' will be populated by the value of this configuration
+option, and the 'os_hash_value' will be populated by the hexdigest computed
+when the algorithm is applied to the uploaded or imported image data.
+
+The value must be a valid secure hash algorithm name recognized by the
+python 'hashlib' library.  You can determine what these are by examining
+the 'hashlib.algorithms_available' data member of the version of the
+library being used in your Glance installation.  For interoperability
+purposes, however, we recommend that you use the set of secure hash
+names supplies by the 'hashlib.algorithms_guaranteed' data member because
+those algorithms are guaranteed to be supported by the 'hashlib' library
+on all platforms.  Thus, any image consumer using 'hashlib' locally should
+be able to verify the 'os_hash_value' of the image.
+
+The default value of 'sha512' is a performant secure hash algorithm.
+
+If this option is misconfigured, any attempts to store image data will fail.
+For that reason, we recommend using the default value.
+
+Possible values:
+    * Any secure hash algorithm name recognized by the Python 'hashlib'
+      library
+
+Related options:
+    * None
+
 """)),
     cfg.IntOpt('image_member_quota', default=128,
                help=_("""

diff --git a/glance/db/__init__.py b/glance/db/__init__.py
@@ -130,6 +130,8 @@ def _format_image_from_db(self, db_image, db_tags):
             protected=db_image['protected'],
             locations=location_strategy.get_ordered_locations(locations),
             checksum=db_image['checksum'],
+            os_hash_algo=db_image['os_hash_algo'],
+            os_hash_value=db_image['os_hash_value'],
             owner=db_image['owner'],
             disk_format=db_image['disk_format'],
             container_format=db_image['container_format'],
@@ -162,6 +164,8 @@ def _format_image_to_db(self, image):
             'protected': image.protected,
             'locations': locations,
             'checksum': image.checksum,
+            'os_hash_algo': image.os_hash_algo,
+            'os_hash_value': image.os_hash_value,
             'owner': image.owner,
             'disk_format': image.disk_format,
             'container_format': image.container_format,

diff --git a/glance/db/simple/api.py b/glance/db/simple/api.py
@@ -225,6 +225,8 @@ def _image_format(image_id, **values):
         'size': None,
         'virtual_size': None,
         'checksum': None,
+        'os_hash_algo': None,
+        'os_hash_value': None,
         'tags': [],
         'created_at': dt,
         'updated_at': dt,
@@ -735,7 +737,7 @@ def image_create(context, image_values, v1_mode=False):
                         'protected', 'is_public', 'container_format',
                         'disk_format', 'created_at', 'updated_at', 'deleted',
                         'deleted_at', 'properties', 'tags', 'visibility',
-                        'os_hidden'])
+                        'os_hidden', 'os_hash_algo', 'os_hash_value'])
 
     incorrect_keys = set(image_values.keys()) - allowed_keys
     if incorrect_keys:

diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/rocky_migrate02_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/rocky_migrate02_empty.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2018 Verizon Wireless
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def has_migrations(engine):
+    """Returns true if at least one data row can be migrated."""
+
+    return False
+
+
+def migrate(engine):
+    """Return the number of rows migrated."""
+
+    return 0
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/rocky_contract02_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/rocky_contract02_empty.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2018 Verizon Wireless
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+# revision identifiers, used by Alembic.
+revision = 'rocky_contract02'
+down_revision = 'rocky_contract01'
+branch_labels = None
+depends_on = 'rocky_expand02'
+
+
+def upgrade():
+    pass
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/rocky_expand02_add_os_hash_.py b/glance/db/sqlalchemy/alembic_migrations/versions/rocky_expand02_add_os_hash_.py
@@ -0,0 +1,33 @@
+# Copyright (C) 2018 Verizon Wireless
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add os_hash_algo and os_hash_value columns to images table"""
+
+from alembic import op
+from sqlalchemy import Column, String
+
+# revision identifiers, used by Alembic.
+revision = 'rocky_expand02'
+down_revision = 'rocky_expand01'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    algo_col = Column('os_hash_algo', String(length=64), nullable=True)
+    value_col = Column('os_hash_value', String(length=128), nullable=True)
+    op.add_column('images', algo_col)
+    op.add_column('images', value_col)
+    op.create_index('os_hash_value_image_idx', 'images', ['os_hash_value'])
diff --git a/glance/db/sqlalchemy/api.py b/glance/db/sqlalchemy/api.py
@@ -468,6 +468,10 @@ def _make_conditions_from_filters(filters, is_public=None):
         checksum = filters.pop('checksum')
         image_conditions.append(models.Image.checksum == checksum)
 
+    if 'os_hash_value' in filters:
+        os_hash_value = filters.pop('os_hash_value')
+        image_conditions.append(models.Image.os_hash_value == os_hash_value)
+
     for (k, v) in filters.pop('properties', {}).items():
         prop_filters = _make_image_property_condition(key=k, value=v)
         prop_conditions.append(prop_filters)