From d09b7b72267982c12b88a0eaa87a4ae4233f8a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charlier?= Date: Thu, 21 Mar 2013 16:21:11 +0100 Subject: [PATCH] Change the column counter_volume to Float MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit counter_volume was previously an Integer (32 bits value in SQL implementations as MySQL or PostgreSQL) and limited in values from -2147483648 to 2147483647, very small for things like disk I/O bytes, network I/O bytes, image size, object size, volume size, … Furthermore one might want to store some real numbers (ℝ) (temperatures, power comsumption, …) Change-Id: Ifdb8e8e84e8e01df267d71bc72cdc3f56bb2f695 Fixes: bug #1158276 --- .../versions/006_counter_volume_is_float.py | 32 ++++++++ ceilometer/storage/sqlalchemy/models.py | 5 +- tests/storage/base.py | 79 +++++++++++++++++++ tests/storage/test_impl_hbase.py | 4 + tests/storage/test_impl_mongodb.py | 4 + tests/storage/test_impl_sqlalchemy.py | 4 + 6 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 ceilometer/storage/sqlalchemy/migrate_repo/versions/006_counter_volume_is_float.py diff --git a/ceilometer/storage/sqlalchemy/migrate_repo/versions/006_counter_volume_is_float.py b/ceilometer/storage/sqlalchemy/migrate_repo/versions/006_counter_volume_is_float.py new file mode 100644 index 0000000000..85eb59ff06 --- /dev/null +++ b/ceilometer/storage/sqlalchemy/migrate_repo/versions/006_counter_volume_is_float.py @@ -0,0 +1,32 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 eNovance SAS +# Author: François Charlier +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * + +meta = MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + meter = Table('meter', meta, autoload=True) + meter.c.counter_volume.alter(type=Float(53)) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + meter = Table('meter', meta, autoload=True) + meter.c.counter_volume.alter(type=Integer) diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index 75625784b0..b545076946 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -22,7 +22,8 @@ import urlparse from oslo.config import cfg -from sqlalchemy import Column, Integer, String, Table, ForeignKey, DateTime +from sqlalchemy import Column, Integer, String, Table, ForeignKey, DateTime, \ + Float from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship from sqlalchemy.types import TypeDecorator, VARCHAR @@ -108,7 +109,7 @@ class Meter(Base): resource_metadata = Column(JSONEncodedDict) counter_type = Column(String(255)) counter_unit = Column(String(255)) - counter_volume = Column(Integer) + counter_volume = Column(Float(53)) timestamp = Column(DateTime, default=timeutils.utcnow) message_signature = Column(String) message_id = Column(String) diff --git a/tests/storage/base.py b/tests/storage/base.py index 6c8abb12b5..91412e713f 100644 --- a/tests/storage/base.py +++ b/tests/storage/base.py @@ -878,3 +878,82 @@ def test_one_resource(self): assert results['max'] == 7 assert results['sum'] == 18 assert results['avg'] == 6 + + +class CounterDataTypeTest(DBTestBase): + + def prepare_data(self): + c = counter.Counter( + 'dummyBigCounter', + counter.TYPE_CUMULATIVE, + unit='', + volume=3372036854775807, + user_id='user-id', + project_id='project-id', + resource_id='resource-id', + timestamp=datetime.datetime(2012, 7, 2, 10, 40), + resource_metadata={} + ) + msg = meter.meter_message_from_counter( + c, + cfg.CONF.metering_secret, + 'test-1', + ) + + self.conn.record_metering_data(msg) + + c = counter.Counter( + 'dummySmallCounter', + counter.TYPE_CUMULATIVE, + unit='', + volume=-3372036854775807, + user_id='user-id', + project_id='project-id', + resource_id='resource-id', + timestamp=datetime.datetime(2012, 7, 2, 10, 40), + resource_metadata={} + ) + msg = meter.meter_message_from_counter( + c, + cfg.CONF.metering_secret, + 'test-1', + ) + self.conn.record_metering_data(msg) + + c = counter.Counter( + 'floatCounter', + counter.TYPE_CUMULATIVE, + unit='', + volume=1938495037.53697, + user_id='user-id', + project_id='project-id', + resource_id='resource-id', + timestamp=datetime.datetime(2012, 7, 2, 10, 40), + resource_metadata={} + ) + msg = meter.meter_message_from_counter( + c, + cfg.CONF.metering_secret, + 'test-1', + ) + self.conn.record_metering_data(msg) + + def test_storage_can_handle_large_values(self): + f = storage.EventFilter( + meter='dummyBigCounter', + ) + results = list(self.conn.get_samples(f)) + self.assertEqual(results[0]['counter_volume'], 3372036854775807) + + f = storage.EventFilter( + meter='dummySmallCounter', + ) + results = list(self.conn.get_samples(f)) + self.assertEqual(results[0]['counter_volume'], -3372036854775807) + + def test_storage_can_handle_float_values(self): + f = storage.EventFilter( + meter='floatCounter', + ) + results = list(self.conn.get_samples(f)) + self.assertEqual(results[0]['counter_volume'], 1938495037.53697) diff --git a/tests/storage/test_impl_hbase.py b/tests/storage/test_impl_hbase.py index 5483d1382b..0d0dfcbd98 100644 --- a/tests/storage/test_impl_hbase.py +++ b/tests/storage/test_impl_hbase.py @@ -177,6 +177,10 @@ class StatisticsTest(base.StatisticsTest, HBaseEngineTestBase): pass +class CounterDataTypeTest(base.CounterDataTypeTest, HBaseEngineTestBase): + pass + + ############### # This is a very crude version of "in-memory HBase", which implements just # enough functionality of HappyBase API to support testing of our driver. diff --git a/tests/storage/test_impl_mongodb.py b/tests/storage/test_impl_mongodb.py index 2c1ab348d7..88e1febdc8 100644 --- a/tests/storage/test_impl_mongodb.py +++ b/tests/storage/test_impl_mongodb.py @@ -226,3 +226,7 @@ def old_record_metering_data(self, data): def test_counter_unit(self): meters = list(self.conn.get_meters()) self.assertEqual(len(meters), 1) + + +class CounterDataTypeTest(base.CounterDataTypeTest, MongoDBEngineTestBase): + pass diff --git a/tests/storage/test_impl_sqlalchemy.py b/tests/storage/test_impl_sqlalchemy.py index 776407c624..84f21fea64 100644 --- a/tests/storage/test_impl_sqlalchemy.py +++ b/tests/storage/test_impl_sqlalchemy.py @@ -135,6 +135,10 @@ class StatisticsTest(base.StatisticsTest, SQLAlchemyEngineTestBase): pass +class CounterDataTypeTest(base.CounterDataTypeTest, SQLAlchemyEngineTestBase): + pass + + def test_model_table_args(): cfg.CONF.database_connection = 'mysql://localhost' assert table_args()