Migrate to fileutils and lockutils.

Migrate nova to using openstack-common's file and lock utilities.
Resolves bug 1063230.

Change-Id: I1a4c87856bc08cd33b61d7098ed856baa4583654

doc/source/devref/threading.rst

@@ -12,7 +12,7 @@ view, each OpenStack service runs in a single thread.
 
 The use of green threads reduces the likelihood of race conditions, but does
 not completely eliminate them. In some cases, you may need to use the
-``@utils.synchronized(...)`` decorator to avoid races.
+``@lockutils.synchronized(...)`` decorator to avoid races.
 
 In addition, since there is only one operating system thread, a call that
 blocks that main thread will block the entire process.

nova/api/openstack/compute/contrib/cloudpipe.py

@@ -25,6 +25,7 @@
 from nova import exception
 from nova import flags
 from nova import network
+from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import timeutils
 from nova import utils
@@ -69,7 +70,7 @@ def setup(self):
         # NOTE(vish): One of the drawbacks of doing this in the api is
         #             the keys will only be on the api node that launched
         #             the cloudpipe.
-        utils.ensure_tree(FLAGS.keys_path)
+        fileutils.ensure_tree(FLAGS.keys_path)
 
     def _get_all_cloudpipes(self, context):
         """Get all cloudpipes"""

nova/cloudpipe/pipelib.py

@@ -33,6 +33,7 @@
 from nova import exception
 from nova import flags
 from nova.openstack.common import cfg
+from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova import utils
 
@@ -150,7 +151,7 @@ def setup_key_pair(self, context):
                                                  key_name)
             private_key = result['private_key']
             key_dir = os.path.join(FLAGS.keys_path, context.user_id)
-            utils.ensure_tree(key_dir)
+            fileutils.ensure_tree(key_dir)
             key_path = os.path.join(key_dir, '%s.pem' % key_name)
             with open(key_path, 'w') as f:
                 f.write(private_key)

nova/compute/manager.py

@@ -64,6 +64,7 @@
 from nova.openstack.common import excutils
 from nova.openstack.common import importutils
 from nova.openstack.common import jsonutils
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova.openstack.common.notifier import api as notifier
 from nova.openstack.common import rpc
@@ -844,7 +845,7 @@ def run_instance(self, context, instance, request_spec=None,
         if injected_files is None:
             injected_files = []
 
-        @utils.synchronized(instance['uuid'])
+        @lockutils.synchronized(instance['uuid'], 'nova-')
         def do_run_instance():
             self._run_instance(context, request_spec,
                     filter_properties, requested_networks, injected_files,
@@ -954,7 +955,7 @@ def terminate_instance(self, context, instance, bdms=None):
         if not bdms:
             bdms = self._get_instance_volume_bdms(context, instance["uuid"])
 
-        @utils.synchronized(instance['uuid'])
+        @lockutils.synchronized(instance['uuid'], 'nova-')
         def do_terminate_instance(instance, bdms):
             try:
                 self._delete_instance(context, instance, bdms)
@@ -2027,7 +2028,7 @@ def _attach_volume_boot(self, context, instance, volume, mountpoint):
     @wrap_instance_fault
     def reserve_block_device_name(self, context, instance, device, volume_id):
 
-        @utils.synchronized(instance['uuid'])
+        @lockutils.synchronized(instance['uuid'], 'nova-')
         def do_reserve():
             result = compute_utils.get_device_name_for_instance(context,
                                                                 instance,

nova/compute/resource_tracker.py

@@ -27,6 +27,7 @@
 from nova.openstack.common import cfg
 from nova.openstack.common import importutils
 from nova.openstack.common import jsonutils
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova import utils
 
@@ -121,7 +122,7 @@ def resource_claim(self, context, instance_ref, limits=None):
         claim = self.begin_resource_claim(context, instance_ref, limits)
         return ResourceContextManager(context, claim, self)
 
-    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    @lockutils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, 'nova-')
     def begin_resource_claim(self, context, instance_ref, limits=None):
         """Indicate that some resources are needed for an upcoming compute
         instance build operation.
@@ -293,7 +294,7 @@ def _can_claim_cpu(self, vcpus, vcpu_limit):
 
         return can_claim_cpu
 
-    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    @lockutils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, 'nova-')
     def finish_resource_claim(self, claim):
         """Indicate that the compute operation that previously claimed the
         resources identified by 'claim' has now completed and the resources
@@ -308,7 +309,7 @@ def finish_resource_claim(self, claim):
         if self.claims.pop(claim.claim_id, None):
             LOG.debug(_("Finishing claim: %s") % claim)
 
-    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    @lockutils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, 'nova-')
     def abort_resource_claim(self, context, claim):
         """Indicate that the operation that claimed the resources identified by
         'claim_id' has either failed or been aborted and the resources are no
@@ -328,7 +329,7 @@ def abort_resource_claim(self, context, claim):
             self._update_usage_from_instance(self.compute_node, claim.instance)
             self._update(context, self.compute_node)
 
-    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    @lockutils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, 'nova-')
     def update_usage(self, context, instance):
         """Update the resource usage and stats after a change in an
         instance
@@ -347,7 +348,7 @@ def update_usage(self, context, instance):
     def disabled(self):
         return self.compute_node is None
 
-    @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
+    @lockutils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, 'nova-')
     def update_available_resource(self, context):
         """Override in-memory calculations of compute node resource usage based
         on data audited from the hypervisor layer.

nova/crypto.py

@@ -33,6 +33,7 @@
 from nova import exception
 from nova import flags
 from nova.openstack.common import cfg
+from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import timeutils
 from nova import utils
@@ -112,7 +113,7 @@ def ensure_ca_filesystem():
                                          'genrootca.sh')
 
         start = os.getcwd()
-        utils.ensure_tree(ca_dir)
+        fileutils.ensure_tree(ca_dir)
         os.chdir(ca_dir)
         utils.execute("sh", genrootca_sh_path)
         os.chdir(start)
@@ -301,7 +302,7 @@ def _sign_csr(csr_text, ca_folder):
         start = os.getcwd()
 
         # Change working dir to CA
-        utils.ensure_tree(ca_folder)
+        fileutils.ensure_tree(ca_folder)
         os.chdir(ca_folder)
         utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
                       './openssl.cnf', '-infiles', inbound)

nova/flags.py

@@ -91,9 +91,6 @@ def _get_my_ip():
     cfg.StrOpt('state_path',
                default='$pybasedir',
                help="Top-level directory for maintaining nova's state"),
-    cfg.StrOpt('lock_path',
-               default='$pybasedir',
-               help='Directory to use for lock files'),
     ]
 
 debug_opts = [

nova/network/linux_net.py

@@ -28,7 +28,9 @@
 from nova import exception
 from nova import flags
 from nova.openstack.common import cfg
+from nova.openstack.common import fileutils
 from nova.openstack.common import importutils
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova import utils
 
@@ -344,7 +346,7 @@ def apply(self):
 
         self._apply()
 
-    @utils.synchronized('iptables', external=True)
+    @lockutils.synchronized('iptables', 'nova-', external=True)
     def _apply(self):
         """Apply the current in-memory set of iptables rules.
 
@@ -791,7 +793,7 @@ def kill_dhcp(dev):
 # NOTE(ja): Sending a HUP only reloads the hostfile, so any
 #           configuration options (like dchp-range, vlan, ...)
 #           aren't reloaded.
-@utils.synchronized('dnsmasq_start')
+@lockutils.synchronized('dnsmasq_start', 'nova-')
 def restart_dhcp(context, dev, network_ref):
     """(Re)starts a dnsmasq server for a given network.
 
@@ -858,7 +860,7 @@ def restart_dhcp(context, dev, network_ref):
     _add_dnsmasq_accept_rules(dev)
 
 
-@utils.synchronized('radvd_start')
+@lockutils.synchronized('radvd_start', 'nova-')
 def update_ra(context, dev, network_ref):
     conffile = _ra_file(dev, 'conf')
     conf_str = """
@@ -957,15 +959,15 @@ def _device_exists(device):
 
 def _dhcp_file(dev, kind):
     """Return path to a pid, leases or conf file for a bridge/device."""
-    utils.ensure_tree(FLAGS.networks_path)
+    fileutils.ensure_tree(FLAGS.networks_path)
     return os.path.abspath('%s/nova-%s.%s' % (FLAGS.networks_path,
                                               dev,
                                               kind))
 
 
 def _ra_file(dev, kind):
     """Return path to a pid or conf file for a bridge/device."""
-    utils.ensure_tree(FLAGS.networks_path)
+    fileutils.ensure_tree(FLAGS.networks_path)
     return os.path.abspath('%s/nova-ra-%s.%s' % (FLAGS.networks_path,
                                               dev,
                                               kind))
@@ -1116,7 +1118,7 @@ def ensure_vlan_bridge(_self, vlan_num, bridge, bridge_interface,
         return interface
 
     @classmethod
-    @utils.synchronized('ensure_vlan', external=True)
+    @lockutils.synchronized('ensure_vlan', 'nova-', external=True)
     def ensure_vlan(_self, vlan_num, bridge_interface, mac_address=None):
         """Create a vlan unless it already exists."""
         interface = 'vlan%s' % vlan_num
@@ -1141,7 +1143,7 @@ def ensure_vlan(_self, vlan_num, bridge_interface, mac_address=None):
         return interface
 
     @classmethod
-    @utils.synchronized('ensure_bridge', external=True)
+    @lockutils.synchronized('ensure_bridge', 'nova-', external=True)
     def ensure_bridge(_self, bridge, interface, net_attrs=None, gateway=True):
         """Create a bridge unless it already exists.
 

nova/network/manager.py

@@ -66,6 +66,7 @@
 from nova.openstack.common import excutils
 from nova.openstack.common import importutils
 from nova.openstack.common import jsonutils
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova.openstack.common.notifier import api as notifier
 from nova.openstack.common import rpc
@@ -854,7 +855,7 @@ def __init__(self, network_driver=None, *args, **kwargs):
     def _import_ipam_lib(self, ipam_lib):
         self.ipam = importutils.import_module(ipam_lib).get_ipam_lib(self)
 
-    @utils.synchronized('get_dhcp')
+    @lockutils.synchronized('get_dhcp', 'nova-')
     def _get_dhcp_ip(self, context, network_ref, host=None):
         """Get the proper dhcp address to listen on."""
         # NOTE(vish): this is for compatibility

nova/objectstore/s3server.py

@@ -46,6 +46,7 @@
 
 from nova import flags
 from nova.openstack.common import cfg
+from nova.openstack.common import fileutils
 from nova import utils
 from nova import wsgi
 
@@ -93,7 +94,7 @@ def __init__(self, root_directory, bucket_depth=0, mapper=None):
         mapper.connect('/{bucket_name}/',
                 controller=lambda *a, **kw: BucketHandler(self)(*a, **kw))
         self.directory = os.path.abspath(root_directory)
-        utils.ensure_tree(self.directory)
+        fileutils.ensure_tree(self.directory)
         self.bucket_depth = bucket_depth
         super(S3Application, self).__init__(mapper)
 
@@ -285,7 +286,7 @@ def put(self, bucket_name):
             os.path.exists(path)):
             self.set_status(403)
             return
-        utils.ensure_tree(path)
+        fileutils.ensure_tree(path)
         self.finish()
 
     def delete(self, bucket_name):
@@ -334,7 +335,7 @@ def put(self, bucket, object_name):
             self.set_status(403)
             return
         directory = os.path.dirname(path)
-        utils.ensure_tree(directory)
+        fileutils.ensure_tree(directory)
         object_file = open(path, "w")
         object_file.write(self.request.body)
         object_file.close()

nova/openstack/common/fileutils.py

@@ -0,0 +1,35 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import errno
+import os
+
+
+def ensure_tree(path):
+    """Create a directory (and any ancestor directories required)
+
+    :param path: Directory to create
+    """
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST:
+            if not os.path.isdir(path):
+                raise
+        else:
+            raise