Merge pull request #487 from sbt/wip/java-7

Upgrading to java 7 and using posix nio API

.travis.yml

@@ -2,11 +2,11 @@ language: scala
 os:
   - linux
 script:
+  - sbt ++$TRAVIS_SCALA_VERSION "test"
   - sbt ++$TRAVIS_SCALA_VERSION "scripted rpm/* debian/* universal/*"
 scala:
   - 2.10.3
 jdk:
-  - openjdk6
   - openjdk7
   - oraclejdk8
 notifications:

build.sbt

@@ -12,7 +12,8 @@ scalacOptions in Compile ++= Seq("-deprecation", "-target:jvm-1.6")
 
 libraryDependencies ++= Seq(
     "org.apache.commons" % "commons-compress" % "1.4.1",
-    "org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar"))
+    "org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar")),
+    "org.scalatest" %% "scalatest" % "2.2.4" % "test"
 )
 
 site.settings

src/main/scala/com/typesafe/sbt/packager/FileUtil.scala

@@ -1,18 +1,61 @@
 package com.typesafe.sbt
 package packager
 
-import java.io.File
-import sbt.Process
+import java.io.{ File, IOException }
+import java.nio.file.{ Paths, Files }
+import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions }
 
+/**
+ * Setting the file permissions
+ */
 object chmod {
+
+  /**
+   * Using java 7 nio API to set the permissions.
+   *
+   * @param file
+   * @param perms in octal format
+   */
   def apply(file: File, perms: String): Unit =
-    Process(Seq("chmod", perms, file.getAbsolutePath)).! match {
-      case 0 => ()
-      case n => sys.error("Error running chmod " + perms + " " + file)
-    }
-  def safe(file: File, perms: String): Unit =
-    try apply(file, perms)
-    catch {
-      case e: RuntimeException => ()
+    try {
+      Files.setPosixFilePermissions(file.toPath, permissions(perms))
+    } catch {
+      case e: IOException => sys.error("Error setting permissions " + perms + " on " + file.getAbsolutePath + ": " + e.getMessage)
     }
+
+}
+
+/**
+ * Converts a octal unix permission representation into
+ * a java `PosiFilePermissions` compatible string.
+ */
+object permissions {
+
+  /**
+   * @param perms in octal format
+   * @return java 7 posix file permissions
+   */
+  def apply(perms: String): java.util.Set[PosixFilePermission] = PosixFilePermissions fromString convert(perms)
+
+  def convert(perms: String): String = {
+    require(perms.length == 4 || perms.length == 3, s"Permissions must have 3 or 4 digits, got [$perms]")
+    // ignore setuid/setguid/sticky bit
+    val i = if (perms.length == 3) 0 else 1
+    val user = Character getNumericValue (perms charAt i)
+    val group = Character getNumericValue (perms charAt i + 1)
+    val other = Character getNumericValue (perms charAt i + 2)
+
+    asString(user) + asString(group) + asString(other)
+  }
+
+  private def asString(perm: Int): String = perm match {
+    case 0 => "---"
+    case 1 => "--x"
+    case 2 => "-w-"
+    case 3 => "-wx"
+    case 4 => "r--"
+    case 5 => "r-x"
+    case 6 => "rw-"
+    case 7 => "rwx"
+  }
 }

src/main/scala/com/typesafe/sbt/packager/universal/Archives.scala

@@ -7,20 +7,44 @@ import sbt._
 /** Helper methods to package up files into compressed archives. */
 object Archives {
 
-  /** Makes a zip file in the given target directory using the given name. */
-  def makeZip(target: File, name: String, mappings: Seq[(File, String)]): File = {
+  /**
+   * Makes a zip file in the given target directory using the given name.
+   *
+   * @param target folder to build package in
+   * @param name of output (without extension)
+   * @param mappings included in the output
+   * @param top level directory
+   * @return zip file
+   */
+  def makeZip(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
     val zip = target / (name + ".zip")
-    // TODO - If mappings already start with the given name, don't add it?
-    val m2 = mappings map { case (f, p) => f -> (name + "/" + p) }
+
+    // add top level directory if defined
+    val m2 = top map { dir =>
+      mappings map { case (f, p) => f -> (dir + "/" + p) }
+    } getOrElse (mappings)
+
     ZipHelper.zip(m2, zip)
     zip
   }
 
-  /** Makes a zip file in the given target directory using the given name. */
-  def makeNativeZip(target: File, name: String, mappings: Seq[(File, String)]): File = {
+  /**
+   * Makes a zip file in the given target directory using the given name.
+   *
+   * @param target folder to build package in
+   * @param name of output (without extension)
+   * @param mappings included in the output
+   * @param top level directory
+   * @return zip file
+   */
+  def makeNativeZip(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
     val zip = target / (name + ".zip")
-    // TODO - If mappings already start with the given name, don't add it?
-    val m2 = mappings map { case (f, p) => f -> (name + "/" + p) }
+
+    // add top level directory if defined
+    val m2 = top map { dir =>
+      mappings map { case (f, p) => f -> (dir + "/" + p) }
+    } getOrElse (mappings)
+
     ZipHelper.zipNative(m2, zip)
     zip
   }
@@ -29,8 +53,14 @@ object Archives {
    * Makes a dmg file in the given target directory using the given name.
    *
    *  Note:  Only works on OSX
+   *
+   *  @param target folder to build package in
+   *  @param name of output (without extension)
+   *  @param mappings included in the output
+   *  @param top level directory : NOT USED
+   *  @return dmg file
    */
-  def makeDmg(target: File, name: String, mappings: Seq[(File, String)]): File = {
+  def makeDmg(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
     val t = target / "dmg"
     val dmg = target / (name + ".dmg")
     if (!t.isDirectory) IO.createDirectory(t)
@@ -113,25 +143,43 @@ object Archives {
   val makeTgz = makeTarball(gzip, ".tgz") _
   val makeTar = makeTarball(identity, ".tar") _
 
-  /** Helper method used to construct tar-related compression functions. */
-  def makeTarball(compressor: File => File, ext: String)(target: File, name: String, mappings: Seq[(File, String)]): File = {
+  /**
+   * Helper method used to construct tar-related compression functions.
+   * @param target folder to build package in
+   * @param name of output (without extension)
+   * @param mappings included in the output
+   * @param top level directory
+   * @return tar file
+   *
+   */
+  def makeTarball(compressor: File => File, ext: String)(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
     val relname = name
     val tarball = target / (name + ext)
     IO.withTemporaryDirectory { f =>
       val rdir = f / relname
-      val m2 = mappings map { case (f, p) => f -> (rdir / name / p) }
+      val m2 = top map { dir =>
+        mappings map { case (f, p) => f -> (rdir / dir / p) }
+      } getOrElse {
+        mappings map { case (f, p) => f -> (rdir / p) }
+      }
+
       IO.copy(m2)
       // TODO - Is this enough?
       for (f <- (m2 map { case (_, f) => f }); if f.getAbsolutePath contains "/bin/") {
         println("Making " + f.getAbsolutePath + " executable")
         f.setExecutable(true, false)
       }
+
       IO.createDirectory(tarball.getParentFile)
-      val distdir = IO.listFiles(rdir).headOption.getOrElse {
-        sys.error("Unable to find tarball in directory: " + rdir.getAbsolutePath + ".\n  This could be an issue with the temporary filesystem used to create tarballs.")
+
+      // all directories that should be zipped
+      val distdirs = top map (_ :: Nil) getOrElse {
+        IO.listFiles(rdir).map(_.getName).toList // no top level dir, use all available
       }
+
       val tmptar = f / (relname + ".tar")
-      Process(Seq("tar", "-pcvf", tmptar.getAbsolutePath, distdir.getName), Some(rdir)).! match {
+
+      Process(Seq("tar", "-pcvf", tmptar.getAbsolutePath) ++ distdirs, Some(rdir)).! match {
         case 0 => ()
         case n => sys.error("Error tarballing " + tarball + ". Exit code: " + n)
       }

src/main/scala/com/typesafe/sbt/packager/universal/Keys.scala

@@ -11,4 +11,5 @@ trait UniversalKeys {
   val stage = TaskKey[File]("stage", "Create a local directory with all the files laid out as they would be in the final distribution.")
   val dist = TaskKey[File]("dist", "Creates the distribution packages.")
   val stagingDirectory = SettingKey[File]("stagingDirectory", "Directory where we stage distributions/releases.")
+  val topLevelDirectory = SettingKey[Option[String]]("topLevelDirectory", "Top level dir in compressed output file.")
 }

src/main/scala/com/typesafe/sbt/packager/universal/UniversalPlugin.scala

@@ -65,6 +65,7 @@ object UniversalPlugin extends AutoPlugin {
     name in UniversalDocs <<= name in Universal,
     name in UniversalSrc <<= name in Universal,
     packageName in Universal <<= packageName,
+    topLevelDirectory := Some((packageName in Universal).value),
     executableScriptName in Universal <<= executableScriptName
   ) ++
     makePackageSettingsForConfig(Universal) ++
@@ -95,12 +96,12 @@ object UniversalPlugin extends AutoPlugin {
     dist
   }
 
-  private type Packager = (File, String, Seq[(File, String)]) => File
+  private type Packager = (File, String, Seq[(File, String)], Option[String]) => File
   /** Creates packaging settings for a given package key, configuration + archive type. */
   private[this] def makePackageSettings(packageKey: TaskKey[File], config: Configuration)(packager: Packager): Seq[Setting[_]] =
     inConfig(config)(Seq(
       mappings in packageKey <<= mappings map checkMappings,
-      packageKey <<= (target, packageName, mappings in packageKey) map packager
+      packageKey <<= (target, packageName, mappings in packageKey, topLevelDirectory) map packager
     ))
 
   /** check that all mapped files actually exist */

src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala

@@ -10,7 +10,20 @@ import org.apache.commons.compress.compressors.{
 }
 import java.util.zip.Deflater
 import org.apache.commons.compress.utils.IOUtils
+import java.nio.file.{ Paths, Files, FileSystems, FileSystem, StandardCopyOption }
+import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions }
+import java.net.URI
+import scala.collection.JavaConverters._
 
+/**
+ *
+ *
+ *
+ * @see http://stackoverflow.com/questions/17888365/file-permissions-are-not-being-preserved-while-after-zip
+ * @see http://stackoverflow.com/questions/3450250/is-it-possible-to-create-a-script-to-save-and-restore-permissions
+ * @see http://stackoverflow.com/questions/1050560/maintain-file-permissions-when-extracting-from-a-zip-file-using-jdk-5-api
+ * @see http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/zipfilesystemprovider.html
+ */
 object ZipHelper {
   case class FileMapping(file: File, name: String, unixMode: Option[Int] = None)
 
@@ -41,10 +54,12 @@ object ZipHelper {
     }
 
   /**
-   * Creates a zip file attempting to give files the appropriate unix permissions using Java 6 APIs.
+   * Creates a zip file with the apache commons compressor library.
+   *
    * Note: This is known to have some odd issues on MacOSX whereby executable permissions
    * are not actually discovered, even though the Info-Zip headers exist and work on
    * many variants of linux.  Yay Apple.
+   *
    * @param sources   The files to include in the zip file.
    * @param outputZip The location of the output file.
    */
@@ -59,29 +74,32 @@ object ZipHelper {
   }
 
   /**
-   * Creates a zip file using the given set of filters
-   * @param sources   The files to include in the zip file.  A File, Location, Permission pairing.
+   * Creates a zip file attempting to give files the appropriate unix permissions using Java 7 APIs.
+   *
+   * @param sources   The files to include in the zip file.
    * @param outputZip The location of the output file.
    */
-  def zipWithPerms(sources: Traversable[(File, String, Int)], outputZip: File): Unit = {
-    val mappings =
-      for {
-        (file, name, perm) <- sources
-      } yield FileMapping(file, name, Some(perm))
-    archive(mappings.toSeq, outputZip)
-  }
+  def zipNIO(sources: Traversable[(File, String)], outputZip: File): Unit = {
+    require(!outputZip.isDirectory, "Specified output file " + outputZip + " is a directory.")
+    val mappings = sources.toSeq.map {
+      case (file, name) => FileMapping(file, name)
+    }
 
-  /**
-   * Replaces windows backslash file separator with a forward slash, this ensures the zip file entry is correct for
-   * any system it is extracted on.
-   * @param path  The path of the file in the zip file
-   */
-  private def normalizePath(path: String) = {
-    val sep = java.io.File.separatorChar
-    if (sep == '/')
-      path
-    else
-      path.replace(sep, '/')
+    // make sure everything is available
+    val outputDir = outputZip.getParentFile
+    IO createDirectory outputDir
+
+    // zipping the sources into the output zip
+    withZipFilesystem(outputZip) { system =>
+      mappings foreach {
+        case FileMapping(dir, name, _) if dir.isDirectory => Files createDirectories (system getPath name)
+        case FileMapping(file, name, _) =>
+          val dest = system getPath name
+          // create parent directories if available
+          Option(dest.getParent) foreach (Files createDirectories _)
+          Files copy (file.toPath, dest, StandardCopyOption.COPY_ATTRIBUTES)
+      }
+    }
   }
 
   private def archive(sources: Seq[FileMapping], outputFile: File): Unit = {
@@ -103,6 +121,9 @@ object ZipHelper {
     }
   }
 
+  /**
+   * using apache commons compress
+   */
   private def withZipOutput(file: File)(f: ZipArchiveOutputStream => Unit): Unit = {
     val zipOut = new ZipArchiveOutputStream(file)
     zipOut setLevel Deflater.BEST_COMPRESSION
@@ -111,4 +132,39 @@ object ZipHelper {
       zipOut.close()
     }
   }
+
+  /**
+   * Replaces windows backslash file separator with a forward slash, this ensures the zip file entry is correct for
+   * any system it is extracted on.
+   * @param path  The path of the file in the zip file
+   */
+  private def normalizePath(path: String) = {
+    val sep = java.io.File.separatorChar
+    if (sep == '/')
+      path
+    else
+      path.replace(sep, '/')
+  }
+
+  /**
+   * Opens a zip filesystem and creates the file if necessary.
+   *
+   * Note: This will override an existing zipFile if existent!
+   *
+   * @param zipFile
+   * @param f: FileSystem => Unit, logic working in the filesystem
+   */
+  def withZipFilesystem(zipFile: File, overwrite: Boolean = true)(f: FileSystem => Unit) {
+    if (overwrite) Files deleteIfExists zipFile.toPath
+    val env = Map("create" -> "true").asJava
+    val uri = URI.create("jar:file:" + zipFile.getAbsolutePath)
+
+    val system = FileSystems.newFileSystem(uri, env)
+    try {
+      f(system)
+    } finally {
+      system.close()
+    }
+  }
+
 }

src/sbt-test/universal/test-zips-no-top-level-dir/build.sbt

@@ -0,0 +1,7 @@
+enablePlugins(JavaAppPackaging)
+
+name := "simple-test"
+
+version := "0.1.0"
+
+topLevelDirectory := None

src/sbt-test/universal/test-zips-no-top-level-dir/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))

src/sbt-test/universal/test-zips-no-top-level-dir/src/universal/conf/test

@@ -0,0 +1 @@
+# Test configuration to include in zips.