21051454 would like 64-bit pkg utilities

src/modules/misc.py

@@ -165,12 +165,14 @@ def copytree(src, dst):
                         os.utime(d_path, (s.st_atime, s.st_mtime))
                 elif S_ISSOCK(s.st_mode):
                         sock = socket.socket(socket.AF_UNIX)
+                        # os.mknod doesn't work correctly in 64 bit.
+                        run_bit = struct.calcsize("P") * 8
                         # The s11 fcs version of python doesn't have os.mknod()
                         # but sock.bind has a path length limitation that we can
                         # hit when archiving the test suite.
                         # E1101 Module '{0}' has no '{1}' member
                         # pylint: disable=E1101
-                        if hasattr(os, "mknod"):
+                        if hasattr(os, "mknod") and run_bit == 32:
                                 os.mknod(d_path, s.st_mode, s.st_dev)
                         else:
                                 try:
@@ -2929,3 +2931,22 @@ def set_fd_limits(printer=None):
                     " and try the requested operation again: {1}")\
                     .format(soft, e))
                 sys.exit(EXIT_OOPS)
+
+def set_memory_limit(bytes, allow_override=True):
+        """Limit memory consumption of current process to 'bytes'."""
+
+        if allow_override:
+                try:
+                        bytes = int(os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"])
+                except (KeyError, ValueError):
+                        pass
+
+        try:
+                resource.setrlimit(resource.RLIMIT_DATA, (bytes, bytes))
+        except AttributeError:
+                # If platform doesn't support RLIMIT_DATA, just ignore it.
+                pass
+        except ValueError:
+                # An unprivileged user can not raise a previously set limit,
+                # if that ever happens, just ignore it.
+                pass

src/pull.py

@@ -398,6 +398,9 @@ def main_func():
 
         temp_root = misc.config_temp_root()
 
+        # set process limits for memory consumption to 8GB
+        misc.set_memory_limit(8 * 1024 * 1024 * 1024)
+
         global_settings.client_name = "pkgrecv"
         target = os.environ.get("PKG_DEST", None)
         src_uri = os.environ.get("PKG_SRC", None)

src/setup.py

@@ -106,6 +106,14 @@
 assert py_version in ('2.6', '2.7')
 py_install_dir = 'usr/lib/python' + py_version + '/vendor-packages'
 
+py64_executable = None
+#Python versions > 3.4 are always 64 bit and located in /usr/bin.
+if float(py_version) <= 3.4 and osname == 'sunos':
+        if arch == 'sparc':
+                py64_executable = '/usr/bin/sparcv9/python' + py_version
+        elif arch == 'i386':
+                py64_executable = '/usr/bin/amd64/python' + py_version
+
 scripts_dir = 'usr/bin'
 lib_dir = 'usr/lib'
 svc_method_dir = 'lib/svc/method'
@@ -681,7 +689,8 @@ def run(self):
                                 dst_path = util.change_root(self.root_dir,
                                        os.path.join(d, dstname))
                                 dir_util.mkpath(dst_dir, verbose=True)
-                                file_util.copy_file(srcname, dst_path, update=True)
+                                file_util.copy_file(srcname, dst_path,
+                                    update=True)
                                 # make scripts executable
                                 os.chmod(dst_path,
                                     os.stat(dst_path).st_mode
@@ -772,6 +781,44 @@ def run_cmd(args, swdir, updenv=None, ignerr=False, savestderr=None):
                 if stderr:
                         stderr.close()
 
+def _copy_file_contents(src, dst, buffer_size=16*1024):
+        """A clone of distutils.file_util._copy_file_contents() that modifies
+        python files as they are installed."""
+
+        # Look for shebang line to replace with arch-specific Python executable.
+        shebang_re = re.compile('^#!.*python[0-9]\.[0-9]')
+        first_buf = True
+
+        with open(src, "rb") as sfp:
+                try:
+                        os.unlink(dst)
+                except EnvironmentError as e:
+                        if e.errno != errno.ENOENT:
+                                raise DistutilsFileError("could not delete "
+                                    "'{0}': {1}".format(dst, e))
+
+                with open(dst, "wb") as dfp:
+                        while True:
+                                buf = sfp.read(buffer_size)
+                                if not buf:
+                                        break
+                                if src.endswith(".py"):
+                                        if not first_buf or not py64_executable:
+                                                dfp.write(buf)
+                                                continue
+
+                                        fl = buf[:buf.find(os.linesep) + 1]
+                                        sb_match = shebang_re.search(fl)
+                                        if sb_match:
+                                                buf = shebang_re.sub(
+                                                    "#!" + py64_executable,
+                                                    buf)
+                                dfp.write(buf)
+                                first_buf = False
+
+# Make file_util use our version of _copy_file_contents
+file_util._copy_file_contents = _copy_file_contents
+
 def intltool_update_maintain():
         """Check if scope of localization looks up-to-date or possibly not,
         by comparing file set described in po/POTFILES.{in,skip} and
@@ -963,7 +1010,7 @@ class installfile(Command):
             ("mode=", "m", "file mode"),
         ]
 
-        description = "De-CDDLing file copy"
+        description = "Modifying file copy"
 
         def initialize_options(self):
                 self.file = None

src/tests/api/t_misc.py

@@ -33,6 +33,7 @@
 import os
 import shutil
 import stat
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -115,6 +116,86 @@ def test_psinfo(self):
                 libc = ctypes.CDLL('libc.so')
                 self.assertEqual(psinfo.pr_zoneid, libc.getzoneid())
 
+        def test_memory_limit(self):
+                """Verify that set_memory_limit works."""
+
+                # memory limit to test, keep small to avoid test slowdown
+                mem_cap = 100 * 1024 * 1024
+                # memory tolerance: allowed discrepancy between set limit and
+                # measured process resources. Note, that we have a static
+                # overhead in waste.py for the forking of ps, so while 20M seems
+                # large compared to a 100M limit, in a real world example with
+                # 8G limit it's fairly small.
+                mem_tol = 20 * 1024 * 1024
+
+                waste_mem_py = """
+import os
+import resource
+import subprocess
+
+import pkg.misc as misc
+
+misc.set_memory_limit({0})
+i = 0
+x = {{}}
+try:
+        while True:
+                i += 1
+                x[i] = range(i)
+except MemoryError:
+        # give us some breathing room (enough so the test with env var works)
+        misc.set_memory_limit({0} * 3, allow_override=False)
+        print subprocess.check_output(['ps', '-o', 'rss=', '-p',
+            str(os.getpid())]).strip()
+""".format(str(mem_cap))
+
+                # Re-setting limits which are higher than original limit can
+                # only be done by root.
+                self.assertTrue(os.geteuid() == 0,
+                    "must be root to run this test")
+
+                tmpdir = tempfile.mkdtemp(dir=self.test_root)
+                tmpfile = os.path.join(tmpdir, 'waste.py')
+                with open(tmpfile, 'w') as f:
+                        f.write(waste_mem_py)
+
+                res = int(subprocess.check_output(['python2.7', tmpfile]))
+                # convert from kB to bytes
+                res *= 1024
+
+                self.debug("mem_cap:   " + str(mem_cap))
+                self.debug("proc size: " + str(res))
+
+                self.assertTrue(res < mem_cap + mem_tol,
+                    "process mem consumption too high")
+                self.assertTrue(res > mem_cap - mem_tol,
+                    "process mem consumption too low")
+
+                # test if env var works
+                os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = str(mem_cap * 2)
+                res = int(subprocess.check_output(['python2.7', tmpfile]))
+                res *= 1024
+
+                self.debug("mem_cap:   " + str(mem_cap))
+                self.debug("proc size: " + str(res))
+
+                self.assertTrue(res < mem_cap * 2 + mem_tol,
+                    "process mem consumption too high")
+                self.assertTrue(res > mem_cap * 2 - mem_tol,
+                    "process mem consumption too low")
+
+                # test if invalid env var is handled correctly
+                os.environ["PKG_CLIENT_MAX_PROCESS_SIZE"] = "octopus"
+                res = int(subprocess.check_output(['python2.7', tmpfile]))
+                res *= 1024
+
+                self.debug("mem_cap:   " + str(mem_cap))
+                self.debug("proc size: " + str(res))
+
+                self.assertTrue(res < mem_cap + mem_tol,
+                    "process mem consumption too high")
+                self.assertTrue(res > mem_cap - mem_tol,
+                    "process mem consumption too low")
 
 if __name__ == "__main__":
         unittest.main()

src/tests/cli/t_pkg_install.py

@@ -3906,15 +3906,6 @@ def test_special_salvage(self):
                 sock = socket.socket(socket.AF_UNIX)
                 sock.bind(os.path.join(self.img_path(), "salvage", "socket"))
                 sock.close()
-                # We also test block and character special files, but only if
-                # os.mknod() is available, which it isn't always.
-                # Since mknod only supports 32-bit integer currently, we have
-                # to check if we are running in 32-bit.
-                run_bit = struct.calcsize("P") * 8
-                if hasattr(os, "mknod") and run_bit == 32:
-                        st = os.stat("/dev/null")
-                        os.mknod(os.path.join(self.img_path(), "salvage",
-                            "node"), st.st_mode, st.st_dev)
 
                 # This could hang reading fifo, or keel over reading socket.
                 self.pkg("uninstall salvage-special")