Skip to content
Permalink
Browse files

Merge branch 'master' into JENKINS-26781 and fixing @SInCE TODO.

  • Loading branch information...
jglick committed Apr 14, 2015
2 parents 37167d5 + e26d6b7 commit b62a07c0dbba637001539c1e668be814a164f701
@@ -54,6 +54,11 @@

<!-- Record your changes in the trunk here. -->
<div id="trunk" style="display:none"><!--=TRUNK-BEGIN=-->
<ul class=image>
<li class=>
</ul>
</div><!--=TRUNK-END=-->
<h3><a name=v1.609>What's new in 1.609</a> (2015/04/12)</h3>
<ul class=image>
<li class=bug>
When concurrent builds are enabled, artifact retention policy may delete artifact being
@@ -62,9 +67,7 @@
<li class=bug>
Documentation for $BUILD_ID did not reflect current reality
(<a href="https://issues.jenkins-ci.org/browse/JENKINS-26520">issue 26520</a>)
<li class=>
</ul>
</div><!--=TRUNK-END=-->
<h3><a name=v1.608>What's new in 1.608</a> (2015/04/05)</h3>
<ul class=image>
<li class=bug>
@@ -5,7 +5,7 @@
<parent>
<groupId>org.jenkins-ci.main</groupId>
<artifactId>pom</artifactId>
<version>1.609-SNAPSHOT</version>
<version>1.610-SNAPSHOT</version>
</parent>

<artifactId>cli</artifactId>
@@ -29,7 +29,7 @@ THE SOFTWARE.
<parent>
<groupId>org.jenkins-ci.main</groupId>
<artifactId>pom</artifactId>
<version>1.609-SNAPSHOT</version>
<version>1.610-SNAPSHOT</version>
</parent>

<artifactId>jenkins-core</artifactId>
@@ -276,6 +276,11 @@ THE SOFTWARE.
<artifactId>commons-beanutils</artifactId>
<version>1.8.3</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
@@ -33,7 +33,6 @@
import hudson.model.Computer;
import hudson.model.Item;
import hudson.model.TaskListener;
import hudson.org.apache.tools.tar.TarInputStream;
import hudson.os.PosixAPI;
import hudson.os.PosixException;
import hudson.remoting.Callable;
@@ -70,7 +69,6 @@
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.zip.ZipEntry;
import org.apache.tools.zip.ZipFile;
import org.kohsuke.stapler.Stapler;
@@ -120,6 +118,8 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import jenkins.security.MasterToSlaveCallable;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.jenkinsci.remoting.RoleChecker;
import org.jenkinsci.remoting.RoleSensitive;

@@ -2266,12 +2266,15 @@ private Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out) t

/**
* Reads from a tar stream and stores obtained files to the base dir.
* @since TODO supports large files > 10 GB, migration to commons-compress
*/
private void readFromTar(String name, File baseDir, InputStream in) throws IOException {
TarInputStream t = new TarInputStream(in);
TarArchiveInputStream t = new TarArchiveInputStream(in);

// TarInputStream t = new TarInputStream(in);
try {
TarEntry te;
while ((te = t.getNextEntry()) != null) {
TarArchiveEntry te;
while ((te = t.getNextTarEntry()) != null) {
File f = new File(baseDir,te.getName());
if(te.isDirectory()) {
mkdirs(f);
@@ -2280,8 +2283,7 @@ private void readFromTar(String name, File baseDir, InputStream in) throws IOExc
if (parent != null) mkdirs(parent);
writing(f);

byte linkFlag = (Byte) LINKFLAG_FIELD.get(te);
if (linkFlag==TarEntry.LF_SYMLINK) {
if (te.isSymbolicLink()) {
new FilePath(f).symlinkTo(te.getLinkName(), TaskListener.NULL);
} else {
IOUtils.copy(t,f);
@@ -2298,8 +2300,6 @@ private void readFromTar(String name, File baseDir, InputStream in) throws IOExc
} catch (InterruptedException e) {
Thread.currentThread().interrupt(); // process this later
throw new IOException("Failed to extract "+name,e);
} catch (IllegalAccessException e) {
throw new IOException("Failed to extract "+name,e);
} finally {
t.close();
}
@@ -2722,20 +2722,6 @@ public int compare(String o1, String o2) {
}
};

private static final Field LINKFLAG_FIELD = getTarEntryLinkFlagField();

private static Field getTarEntryLinkFlagField() {
try {
Field f = TarEntry.class.getDeclaredField("linkFlag");
f.setAccessible(true);
return f;
} catch (SecurityException e) {
throw new AssertionError(e);
} catch (NoSuchFieldException e) {
throw new AssertionError(e);
}
}

/**
* Gets the {@link FilePath} representation of the "~" directory
* (User's home directory in the Unix sense) of the given channel.
@@ -933,7 +933,7 @@ private URL getStaticHelpUrl(Klass<?> c, String suffix) {
/**
* Finds a descriptor from a collection by its ID.
* @param id should match {@link #getId}
* @since TODO
* @since 1.610
*/
public static @CheckForNull <T extends Descriptor> T findById(Collection<? extends T> list, String id) {
for (T d : list) {
@@ -958,7 +958,7 @@ private URL getStaticHelpUrl(Klass<?> c, String suffix) {
/**
* Finds a descriptor from a collection by the class name of the {@link Describable} it describes.
* @param className should match {@link Class#getName} of a {@link #clazz}
* @since TODO
* @since 1.610
*/
public static @CheckForNull <T extends Descriptor> T findByDescribableClassName(Collection<? extends T> list, String className) {
for (T d : list) {
@@ -37,8 +37,9 @@
* methods are provided to position at each successive entry in
* the archive, and the read each entry as a normal input stream
* using read().
*
* @deprecated Use {@link org.apache.commons.compress.archivers.tar.TarArchiveInputStream} instead
*/
@Deprecated
public class TarInputStream extends FilterInputStream {

// CheckStyle:VisibilityModifier OFF - bc
@@ -35,8 +35,11 @@
* The TarOutputStream writes a UNIX tar archive as an OutputStream.
* Methods are provided to put entries, and then write their contents
* by writing to this stream using write().
*
* @deprecated Use {@link org.apache.commons.compress.archivers.tar.TarArchiveOutputStream} instead
*
*/
@Deprecated
public class TarOutputStream extends FilterOutputStream {
/** Fail if a long file name is required in the archive. */
public static final int LONGFILE_ERROR = 0;
@@ -129,6 +129,7 @@ protected Downloadable createDownloadable() {
* This ID needs to be unique, and needs to match the ID token in the JSON update file.
* <p>
* By default we use the fully-qualified class name of the {@link DownloadFromUrlInstaller} subtype.
* @since 1.610
*/
public String getDownloadableId() {
return clazz.getName().replace('$','.');
@@ -37,6 +37,8 @@
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;

import static org.apache.tools.tar.TarConstants.LF_SYMLINK;

@@ -47,24 +49,25 @@
*/
final class TarArchiver extends Archiver {
private final byte[] buf = new byte[8192];
private final TarOutputStream tar;
private final TarArchiveOutputStream tar;

TarArchiver(OutputStream out) {
tar = new TarOutputStream(new BufferedOutputStream(out) {
tar = new TarArchiveOutputStream(new BufferedOutputStream(out) {
// TarOutputStream uses TarBuffer internally,
// which flushes the stream for each block. this creates unnecessary
// data stream fragmentation, and flush request to a remote, which slows things down.
@Override
public void flush() throws IOException {
// so don't do anything in flush
}
});
tar.setLongFileMode(TarOutputStream.LONGFILE_GNU);
});
tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
}

@Override
public void visitSymlink(File link, String target, String relativePath) throws IOException {
TarEntry e = new TarEntry(relativePath, LF_SYMLINK);
TarArchiveEntry e = new TarArchiveEntry(relativePath, LF_SYMLINK);
try {
int mode = IOUtils.mode(link);
if (mode != -1) {
@@ -73,16 +76,11 @@ public void visitSymlink(File link, String target, String relativePath) throws I
} catch (PosixException x) {
// ignore
}

e.setLinkName(target);

try {
StringBuffer linkName = (StringBuffer) LINKNAME_FIELD.get(e);
linkName.setLength(0);
linkName.append(target);
} catch (IllegalAccessException x) {
throw new IOException("Failed to set linkName", x);
}

tar.putNextEntry(e);
tar.putArchiveEntry(e);
tar.closeArchiveEntry();
entriesWritten++;
}

@@ -97,14 +95,14 @@ public void visit(File file, String relativePath) throws IOException {

if(file.isDirectory())
relativePath+='/';
TarEntry te = new TarEntry(relativePath);
TarArchiveEntry te = new TarArchiveEntry(relativePath);
int mode = IOUtils.mode(file);
if (mode!=-1) te.setMode(mode);
te.setModTime(file.lastModified());
if(!file.isDirectory())
te.setSize(file.length());

tar.putNextEntry(te);
tar.putArchiveEntry(te);

if (!file.isDirectory()) {
FileInputStream in = new FileInputStream(file);
@@ -117,25 +115,11 @@ public void visit(File file, String relativePath) throws IOException {
}
}

tar.closeEntry();
tar.closeArchiveEntry();
entriesWritten++;
}

public void close() throws IOException {
tar.close();
}

private static final Field LINKNAME_FIELD = getTarEntryLinkNameField();

private static Field getTarEntryLinkNameField() {
try {
Field f = TarEntry.class.getDeclaredField("linkName");
f.setAccessible(true);
return f;
} catch (SecurityException e) {
throw new AssertionError(e);
} catch (NoSuchFieldException e) {
throw new AssertionError(e);
}
}
}
@@ -4,7 +4,6 @@
import hudson.ExtensionList;
import hudson.ExtensionPoint;
import hudson.TcpSlaveAgentListener;
import hudson.model.AperiodicWork;
import jenkins.model.Jenkins;

import java.io.IOException;
@@ -40,7 +39,7 @@
public abstract void handle(Socket socket) throws IOException, InterruptedException;

/**
* Returns all the registered {@link AperiodicWork}s.
* Returns all the registered {@link AgentProtocol}s.
*/
public static ExtensionList<AgentProtocol> all() {
return ExtensionList.lookup(AgentProtocol.class);
@@ -35,6 +35,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -56,6 +57,8 @@
import org.apache.tools.ant.Project;
import org.apache.tools.ant.taskdefs.Chmod;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -325,6 +328,53 @@ private FilePath createFilePath(final File base, final String... path) throws IO
FileUtils.touch(building);
return new FilePath(building);
}

/**
* Performs round-trip archiving for Tar handling methods.
* @throws Exception test failure
*/
@Test public void compressTarUntarRoundTrip() throws Exception {
checkTarUntarRoundTrip("compressTarUntarRoundTrip_zero", 0);
checkTarUntarRoundTrip("compressTarUntarRoundTrip_small", 100);
checkTarUntarRoundTrip("compressTarUntarRoundTrip_medium", 50000);
}

/**
* Checks that big files (>8GB) can be archived and then unpacked.
* This test is disabled by default due the impact on RAM.
* The actual file size limit is 8589934591 bytes.
* @throws Exception test failure
*/
@Issue("JENKINS-10629")
@Ignore
@Test public void archiveBigFile() throws Exception {
final long largeFileSize = 9000000000L; // >8589934591 bytes
final String filePrefix = "JENKINS-10629";
checkTarUntarRoundTrip(filePrefix, largeFileSize);
}

private void checkTarUntarRoundTrip(String filePrefix, long fileSize) throws Exception {
final File tmpDir = temp.newFolder(filePrefix);
final File tempFile = new File(tmpDir, filePrefix + ".log");
RandomAccessFile file = new RandomAccessFile(tempFile, "rw");
final File tarFile = new File(tmpDir, filePrefix + ".tar");

file.setLength(fileSize);
assumeTrue(fileSize == file.length());
file.close();

// Compress archive
final FilePath tmpDirPath = new FilePath(tmpDir);
int tar = tmpDirPath.tar(new FileOutputStream(tarFile), tempFile.getName());
assertEquals("One file should have been compressed", 1, tar);

// Decompress
FilePath outDir = new FilePath(temp.newFolder(filePrefix + "_out"));
final FilePath outFile = outDir.child(tempFile.getName());
tmpDirPath.child( filePrefix + ".tar").untar(outDir, TarCompression.NONE);
assertEquals("Result file after the roundtrip differs from the initial file",
new FilePath(tempFile).digest(), outFile.digest());
}

@Test public void list() throws Exception {
File baseDir = temp.getRoot();

0 comments on commit b62a07c

Please sign in to comment.
You can’t perform that action at this time.