Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Splitblank #86

Closed
wants to merge 5 commits into from

4 participants

@mhpx

Added feature to specify a separate location for the blankspace partition file in the control/etc/workspace-control/images.conf
"blankspacedir"
left the default value in conf to be secureimages
Feature requested from user due to disk performance/space issues on cluster where large amounts of blankspace were needed and the local filesystem only had space for the image files, and using a networked filesystem for both blankspace and instance caused poor IO on the network filesystem.

mhpx added some commits
@mhpx mhpx Added new config param to images.conf - blankspacedir
Will to used to specify a separate location for the blankspace partition
e76e91f
@mhpx mhpx Initial work to enable having blankspace partition in a separate loca…
…tion. Untested.

On a cloud setup where large amounts of blankspace are required it
was deemed desireable to have the image directory split from the blankspace file
this would hopefully allow the IO load to be spread out.
4a5463b
@mhpx mhpx Fixed some typos, missing refs, etc.
Blankspace partition now being created in configured location
but fails to do mountcopy tasks
222dac7
@mhpx mhpx Changed back to path from blankspacepath - misunderstanding of how th…
…e lf.path was being used.
b1bc0a6
@mhpx mhpx fixed typo in shutil 9895598
@igable
Collaborator

This will be extremely useful for our CANFAR users and their overtaxed GPFS filesystem. Mike you tested this right?

@mhpx

Tested both with having the blankspacedir the same as securelocaldir and having it as a different location
Edit* also tested with having a lustreFS as the blankspacedir

@oldpatricka
Collaborator

This looks good. We probably can't integrate it into 2.9 (We're on the second release candidate, but it should work fine with a 2.9 service.

Maybe one 2.9 comes out, CANFAR can run that, and then later we can look at a point release?

@priteau priteau was assigned
@priteau priteau referenced this pull request from a commit
@mhpx mhpx Enable having blankspace partition in a separate location
On a cloud setup where large amounts of blankspace are required it was
deemed desireable to have the image directory split from the blankspace
file. This would hopefully allow the IO load to be spread out.

Closes #83 and #86.
b3c469d
@priteau priteau closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jan 12, 2012
  1. @mhpx

    Added new config param to images.conf - blankspacedir

    mhpx authored
    Will to used to specify a separate location for the blankspace partition
  2. @mhpx

    Initial work to enable having blankspace partition in a separate loca…

    mhpx authored
    …tion. Untested.
    
    On a cloud setup where large amounts of blankspace are required it
    was deemed desireable to have the image directory split from the blankspace file
    this would hopefully allow the IO load to be spread out.
  3. @mhpx

    Fixed some typos, missing refs, etc.

    mhpx authored
    Blankspace partition now being created in configured location
    but fails to do mountcopy tasks
Commits on Jan 13, 2012
  1. @mhpx

    Changed back to path from blankspacepath - misunderstanding of how th…

    mhpx authored
    …e lf.path was being used.
  2. @mhpx

    fixed typo in shutil

    mhpx authored
This page is out of date. Refresh to see the latest.
View
21 control/etc/workspace-control/images.conf
@@ -24,14 +24,23 @@ localdir: images
securelocaldir: secureimages
+# blankspace partition directory
+# - this directory is where instance specific images that use blankspace
+# will create their partition
+# - this can be the same or separate from the securelocaldir for performance
+# - if this setting is a relative path, it will be resolved from the
+# workspace-control specific var directory (see "dirs.conf")
+
+blankspacedir: secureimages
+
# blankcreate script
-# - this is called when making a blankspace partition. This is not called
-# under sudo. It's just separated into a script for easier replacement or
-# customization.
-# - if this setting is a relative path, it will be resolved from the
+# - this is called when making a blankspace partition. This is not called
+# under sudo. It's just separated into a script for easier replacement or
+# customization.
+# - if this setting is a relative path, it will be resolved from the
# workspace-control specific libexec directory (see "dirs.conf")
-# - if this setting is missing or blank, the blankspace functionality will
-# be disabled
+# - if this setting is missing or blank, the blankspace functionality will
+# be disabled
blankcreate: blankcreate.sh
View
74 control/src/python/workspacecontrol/defaults/imageprocurement/propagate_common.py
@@ -43,6 +43,7 @@ def __init__(self, params, common):
self.c = common
self.localdir = None
self.securelocaldir = None
+ self.blankspacedir = None
self.adapters = None # dict: {keyword: instance of PropagationAdapter}
@@ -56,7 +57,8 @@ def validate(self):
self.blankcreate_path = None
self._validate_blankspacecreate()
-
+ self.blankspacedir = self._validate_blankspacedir("blankspacedir")
+
self.adapters = {}
cp_path = self.p.get_conf_or_none("propagation", "cp")
@@ -184,7 +186,44 @@ def _validate_securelocaldir(self, confname):
self.c.log.debug("secure image directory (per-instance images): %s" % self.securelocaldir)
return securelocaldir
-
+
+ def _validate_blankspacedir(self, confname):
+ blankspacedir = self.p.get_conf_or_none("images", confname)
+ if not blankspacedir:
+ raise InvalidConfig("no images->%s configuration" % (confname))
+ # Alternatively
+ #blankspacedir = self.securelocaldir
+
+ if not os.path.isabs(blankspacedir):
+ blankspacedir = self.c.resolve_var_dir(blankspacedir)
+
+ if not os.path.exists(blankspacedir):
+ self.c.log.warn("%s is configured, but '%s' does not"
+ " exist on the filesystem, attemping to create "
+ " it" % (confname, blankspacedir))
+ try:
+ os.mkdir(blankspacedir)
+ os.chmod(blankspacedir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+ except:
+ exception_type = sys.exc_type
+ try:
+ exceptname = exception_type.__name__
+ except AttributeError:
+ exceptname = exception_type
+ raise InvalidConfig("Problem creating %s: %s: %s"
+ % (confname, str(exceptname), str(sys.exc_value)))
+ self.c.log.warn("created blankspace dir '%s'" % blankspacedir)
+
+ x = os.access(blankspacedir, os.W_OK | os.X_OK | os.R_OK)
+ if x:
+ self.c.log.debug("'%s' exists on the filesystem and is "
+ "rwx-able" % blankspacedir)
+ else:
+ raise InvalidConfig("'%s' exists on the filesystem but is not rwx"
+ % blankspacedir)
+ self.c.log.debug("blankspace partition directory (per-instance blankspace): %s" % self.blankspacedir)
+ return blankspacedir
+
def _validate_blankspacecreate(self):
blankcreate_path = self.p.get_conf_or_none("images", "blankcreate")
if not blankcreate_path:
@@ -271,6 +310,7 @@ def obtain(self):
if action == ACTIONS.CREATE:
if self._is_blankspace_needed(l_files):
+ self._ensure_blankspace_dir()
self._blankspace(l_files)
local_file_set_cls = self.c.get_class_by_keyword("LocalFileSet")
@@ -296,6 +336,7 @@ def process_after_shutdown(self, local_file_set):
self._unpropagate(l_files)
self._destroy_instance_dir()
+ self._destroy_blankspace_dir()
# --------------------------------------------------------------------------
@@ -314,6 +355,7 @@ def process_after_destroy(self, local_file_set):
"""
self._destroy_instance_dir()
+ self._destroy_blankspace_dir()
# --------------------------------------------------------------------------
@@ -359,6 +401,13 @@ def _derive_instance_dir(self):
raise InvalidInput("The %s argument is required." % wc_args.NAME.long_syntax)
vm_securedir = os.path.join(self.securelocaldir, vm_name)
return vm_securedir
+
+ def _derive_blankspace_dir(self):
+ vm_name = self.p.get_arg_or_none(wc_args.NAME)
+ if not vm_name:
+ raise InvalidInput("The %s argument is required." % wc_args.NAME.long_syntax)
+ vm_blankspacedir = os.path.join(self.blankspacedir, vm_name)
+ return vm_blankspacedir
# --------------------------------------------------------------------------
# IMPLs for actual actions the module takes
@@ -380,6 +429,23 @@ def _destroy_instance_dir(self):
self.c.log.debug("Destroying %s" % vmdir)
shutil.rmtree(vmdir)
self.c.log.info("Destroyed VM's unique directory: %s" % vmdir)
+
+ def _ensure_blankspace_dir(self):
+ blankspacedir = self._derive_blankspace_dir()
+ if os.path.exists(blankspacedir):
+ return
+ self.c.log.info("Creating VM's blankspace directory: %s" % blankspacedir)
+ os.mkdir(blankspacedir)
+ os.chmod(blankspacedir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+ self.c.log.debug("created %s" % blankspacedir)
+
+ def _destroy_blankspace_dir(self):
+ blankspacedir = self._derive_blankspace_dir()
+ if not os.path.exists(blankspacedir):
+ return
+ self.c.log.debug("Destroying %s" % blankspacedir)
+ shutil.rmtree(blankspacedir)
+ self.c.log.info("Destroyed VM's unique blankspace directory: %s" % blankspacedir)
def _propagate(self, l_files):
cache_key = self.p.get_arg_or_none(wc_args.CACHECKSUM)
@@ -745,8 +811,8 @@ def _one_imagestr_blankcreate(self, lf, imgstr):
lf._blankspace = int(size)
except:
raise InvalidInput("blank partition name is expected to have embedded size")
-
- lf.path = self._derive_instance_dir()
+
+ lf.path = self._derive_blankspace_dir()
lf.path = os.path.join(lf.path, blank_filename)
if os.path.exists(lf.path):
Something went wrong with that request. Please try again.