Permalink
Browse files

Decodes input and encodes output

Currently novaclient doesn't handle properly incoming and outgoing
encode / decode process. As a solution for this, this patch implements a
decoding process for all data incoming from the user side and decodes
everything going out of the client, i.e: http requests, prints, etc.

This patch introduces a new module (strutils.py) taken from
oslo-incubator in order to use 2 of the functions present in it:

About safe_(decode|encode):

    Both functions try to encode / decode the incoming text using the
    stdin encoding, fallback to python's default encoding if that
    returns None or to UTF-8 as the last option.

    In both functions only basestring objects are accepted and they both
    raise TypeError if an object of another type is passed.

About the general novaclient changes:

    In order to better support non-ASCII characters, it is a good
    practice to use unicode interanlly and encode everything that has to
    go out. This patch aims to do that and introduces this behaviour in
    the client.

Testing:

    A good test (besides using tox) is to use nova client with and
    without setting any locale (export LANG=).

Fixes bug: 1061156

Change-Id: I20b75e42b0c3dac89f1048faa1127253a64f86c7
  • Loading branch information...
1 parent 39b6e00 commit 47e6bc25ae647c9b83b394cfc5734490855acbe2 @flaper87 flaper87 committed Mar 4, 2013
Showing with 147 additions and 11 deletions.
  1. +133 −0 novaclient/openstack/common/strutils.py
  2. +3 −2 novaclient/shell.py
  3. +7 −6 novaclient/utils.py
  4. +3 −2 novaclient/v1_1/base.py
  5. +1 −1 openstack-common.conf
@@ -0,0 +1,133 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import logging
+import sys
+
+LOG = logging.getLogger(__name__)
+
+
+def int_from_bool_as_string(subject):
+ """
+ Interpret a string as a boolean and return either 1 or 0.
+
+ Any string value in:
+
+ ('True', 'true', 'On', 'on', '1')
+
+ is interpreted as a boolean True.
+
+ Useful for JSON-decoded stuff and config file parsing
+ """
+ return bool_from_string(subject) and 1 or 0
+
+
+def bool_from_string(subject):
+ """
+ Interpret a string as a boolean.
+
+ Any string value in:
+
+ ('True', 'true', 'On', 'on', 'Yes', 'yes', '1')
+
+ is interpreted as a boolean True.
+
+ Useful for JSON-decoded stuff and config file parsing
+ """
+ if isinstance(subject, bool):
+ return subject
+ if isinstance(subject, basestring):
+ if subject.strip().lower() in ('true', 'on', 'yes', '1'):
+ return True
+ return False
+
+
+def safe_decode(text, incoming=None, errors='strict'):
+ """
+ Decodes incoming str using `incoming` if they're
+ not already unicode.
+
+ :param incoming: Text's current encoding
+ :param errors: Errors handling policy. See here for valid
+ values http://docs.python.org/2/library/codecs.html
+ :returns: text or a unicode `incoming` encoded
+ representation of it.
+ :raises TypeError: If text is not an isntance of basestring
+ """
+ if not isinstance(text, basestring):
+ raise TypeError("%s can't be decoded" % type(text))
+
+ if isinstance(text, unicode):
+ return text
+
+ if not incoming:
+ incoming = (sys.stdin.encoding or
+ sys.getdefaultencoding())
+
+ try:
+ return text.decode(incoming, errors)
+ except UnicodeDecodeError:
+ # Note(flaper87) If we get here, it means that
+ # sys.stdin.encoding / sys.getdefaultencoding
+ # didn't return a suitable encoding to decode
+ # text. This happens mostly when global LANG
+ # var is not set correctly and there's no
+ # default encoding. In this case, most likely
+ # python will use ASCII or ANSI encoders as
+ # default encodings but they won't be capable
+ # of decoding non-ASCII characters.
+ #
+ # Also, UTF-8 is being used since it's an ASCII
+ # extension.
+ return text.decode('utf-8', errors)
+
+
+def safe_encode(text, incoming=None,
+ encoding='utf-8', errors='strict'):
+ """
+ Encodes incoming str/unicode using `encoding`. If
+ incoming is not specified, text is expected to
+ be encoded with current python's default encoding.
+ (`sys.getdefaultencoding`)
+
+ :param incoming: Text's current encoding
+ :param encoding: Expected encoding for text (Default UTF-8)
+ :param errors: Errors handling policy. See here for valid
+ values http://docs.python.org/2/library/codecs.html
+ :returns: text or a bytestring `encoding` encoded
+ representation of it.
+ :raises TypeError: If text is not an isntance of basestring
+ """
+ if not isinstance(text, basestring):
+ raise TypeError("%s can't be encoded" % type(text))
+
+ if not incoming:
+ incoming = (sys.stdin.encoding or
@marcinkuzminski
marcinkuzminski Mar 7, 2014

I think this assumes sys.stding is actually present, in case you use only pure python api, or in test env this might not be present. I think there should be additional check for encoding being a valid attributed in sys.stdin

+ sys.getdefaultencoding())
+
+ if isinstance(text, unicode):
+ return text.encode(encoding, errors)
+ elif text and encoding != incoming:
+ # Decode text before encoding it with `encoding`
+ text = safe_decode(text, incoming, errors)
+ return text.encode(encoding, errors)
+
+ return text
View
@@ -49,6 +49,7 @@
from novaclient import client
from novaclient import exceptions as exc
import novaclient.extension
+from novaclient.openstack.common import strutils
from novaclient import utils
from novaclient.v1_1 import shell as shell_v1_1
@@ -738,11 +739,11 @@ def start_section(self, heading):
def main():
try:
- OpenStackComputeShell().main(sys.argv[1:])
+ OpenStackComputeShell().main(map(strutils.safe_decode, sys.argv[1:]))
except Exception as e:
logger.debug(e, exc_info=1)
- print("ERROR: %s" % unicode(e), file=sys.stderr)
+ print("ERROR: %s" % strutils.safe_encode(unicode(e)), file=sys.stderr)
sys.exit(1)
View
@@ -6,6 +6,7 @@
import prettytable
from novaclient import exceptions
+from novaclient.openstack.common import strutils
def arg(*args, **kwargs):
@@ -139,7 +140,7 @@ def pretty_choice_list(l):
def print_list(objs, fields, formatters={}, sortby_index=0):
- if sortby_index == None:
+ if sortby_index is None:
sortby = None
else:
sortby = fields[sortby_index]
@@ -162,9 +163,9 @@ def print_list(objs, fields, formatters={}, sortby_index=0):
pt.add_row(row)
if sortby is not None:
- print(pt.get_string(sortby=sortby))
+ print(strutils.safe_encode(pt.get_string(sortby=sortby)))
else:
- print(pt.get_string())
+ print(strutils.safe_encode(pt.get_string()))
def print_dict(d, dict_property="Property"):
@@ -184,7 +185,7 @@ def print_dict(d, dict_property="Property"):
col1 = ''
else:
pt.add_row([k, v])
- print(pt.get_string())
+ print(strutils.safe_encode(pt.get_string()))
def find_resource(manager, name_or_id):
@@ -203,9 +204,9 @@ def find_resource(manager, name_or_id):
# now try to get entity as uuid
try:
- uuid.UUID(str(name_or_id))
+ uuid.UUID(strutils.safe_encode(name_or_id))
return manager.get(name_or_id)
- except (ValueError, exceptions.NotFound):
+ except (TypeError, ValueError, exceptions.NotFound):
pass
# for str id which is not uuid (for Flavor search currently)
@@ -18,6 +18,7 @@
import base64
from novaclient import base
+from novaclient.openstack.common import strutils
# FIXME(sirp): Now that v1_0 has been removed, this can be merged with
@@ -72,8 +73,8 @@ def _boot(self, resource_url, response_key, name, image, flavor,
if userdata:
if hasattr(userdata, 'read'):
userdata = userdata.read()
- elif isinstance(userdata, unicode):
- userdata = userdata.encode('utf-8')
+
+ userdata = strutils.safe_encode(userdata)
body["server"]["user_data"] = base64.b64encode(userdata)
if meta:
body["server"]["metadata"] = meta
@@ -1,7 +1,7 @@
[DEFAULT]
# The list of modules to copy from openstack-common
-modules=setup,timeutils
+modules=setup,timeutils,strutils
# The base module to hold the copy of openstack.common
base=novaclient

0 comments on commit 47e6bc2

Please sign in to comment.