@@ -24,13 +24,24 @@
package io.jenkins.plugins.artifact_manager_jclouds ;
import com.github.rholder.retry.Attempt ;
import com.github.rholder.retry.AttemptTimeLimiters ;
import hudson.AbortException ;
import hudson.EnvVars ;
import hudson.FilePath ;
import hudson.Launcher ;
import hudson.Util ;
import hudson.model.BuildListener ;
import hudson.model.Run ;
import hudson.model.TaskListener ;
import hudson.remoting.VirtualChannel ;
import hudson.slaves.WorkspaceList ;
import hudson.util.DirScanner ;
import hudson.util.io.ArchiverFactory ;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod ;
import io.jenkins.plugins.httpclient.RobustHTTPClient ;
import java.io.File ;
import java.io.IOException ;
import java.io.InputStream ;
import java.io.OutputStream ;
import java.net.HttpURLConnection ;
import java.net.URL ;
import java.nio.file.Files ;
import java.nio.file.InvalidPathException ;
@@ -39,47 +50,19 @@
import java.util.HashMap ;
import java.util.Iterator ;
import java.util.Map ;
import java.util.concurrent.ExecutionException ;
import java.util.logging.Level ;
import java.util.logging.Logger ;
import jenkins.MasterToSlaveFileCallable ;
import jenkins.model.ArtifactManager ;
import jenkins.util.VirtualFile ;
import org.apache.http.client.methods.HttpGet ;
import org.jclouds.blobstore.BlobStore ;
import org.jclouds.blobstore.BlobStoreContext ;
import org.jclouds.blobstore.domain.Blob ;
import org.jclouds.blobstore.domain.StorageMetadata ;
import org.jclouds.blobstore.options.CopyOptions ;
import org.jclouds.blobstore.options.ListContainerOptions ;
import org.jenkinsci.plugins.workflow.flow.StashManager ;
import com.github.rholder.retry.RetryException ;
import com.github.rholder.retry.RetryListener ;
import com.github.rholder.retry.RetryerBuilder ;
import com.github.rholder.retry.StopStrategies ;
import com.github.rholder.retry.WaitStrategies ;
import com.google.common.util.concurrent.UncheckedTimeoutException ;
import hudson.AbortException ;
import hudson.EnvVars ;
import hudson.FilePath ;
import hudson.Launcher ;
import hudson.Util ;
import hudson.model.BuildListener ;
import hudson.model.Computer ;
import hudson.model.Run ;
import hudson.model.TaskListener ;
import hudson.remoting.VirtualChannel ;
import hudson.slaves.WorkspaceList ;
import hudson.util.DirScanner ;
import hudson.util.io.ArchiverFactory ;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod ;
import java.util.concurrent.ExecutorService ;
import java.util.concurrent.Executors ;
import java.util.concurrent.TimeUnit ;
import java.util.concurrent.atomic.AtomicReference ;
import jenkins.MasterToSlaveFileCallable ;
import jenkins.model.ArtifactManager ;
import jenkins.util.JenkinsJVM ;
import jenkins.util.VirtualFile ;
import org.apache.commons.io.IOUtils ;
import org.kohsuke.accmod.Restricted ;
import org.kohsuke.accmod.restrictions.NoExternalUse ;
@@ -91,6 +74,8 @@
private static final Logger LOGGER = Logger . getLogger(JCloudsArtifactManager . class. getName());
static RobustHTTPClient client = new RobustHTTPClient ();
private final BlobStoreProvider provider;
private transient String key; // e.g. myorg/myrepo/master/123
@@ -148,11 +133,8 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene
private final Map<String , URL > artifactUrls; // e.g. "target/x.war", "http://..."
private final TaskListener listener;
// Bind when constructed on the master side; on the agent side, deserialize those values.
private final int stopAfterAttemptNumber = UPLOAD_STOP_AFTER_ATTEMPT_NUMBER ;
private final long waitMultiplier = UPLOAD_WAIT_MULTIPLIER ;
private final long waitMaximum = UPLOAD_WAIT_MAXIMUM ;
private final long timeout = UPLOAD_TIMEOUT ;
// Bind when constructed on the master side; on the agent side, deserialize the same configuration.
private final RobustHTTPClient client = JCloudsArtifactManager . client;
UploadToBlobStorage (Map<String , URL > artifactUrls , TaskListener listener ) {
this . artifactUrls = artifactUrls;
@@ -162,9 +144,7 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene
@Override
public Void invoke (File f , VirtualChannel channel ) throws IOException , InterruptedException {
for (Map . Entry<String , URL > entry : artifactUrls. entrySet()) {
Path local = f. toPath(). resolve(entry. getKey());
URL url = entry. getValue();
uploadFile(local, url, listener, stopAfterAttemptNumber, waitMultiplier, waitMaximum, timeout);
client. uploadFile(new File (f, entry. getKey()), entry. getValue(), listener);
}
return null ;
}
@@ -229,10 +209,7 @@ public void stash(String name, FilePath workspace, Launcher launcher, EnvVars en
private final boolean useDefaultExcludes;
private final String tempDir;
private final TaskListener listener;
private final int stopAfterAttemptNumber = UPLOAD_STOP_AFTER_ATTEMPT_NUMBER ;
private final long waitMultiplier = UPLOAD_WAIT_MULTIPLIER ;
private final long waitMaximum = UPLOAD_WAIT_MAXIMUM ;
private final long timeout = UPLOAD_TIMEOUT ;
private final RobustHTTPClient client = JCloudsArtifactManager . client;
Stash (URL url , String includes , String excludes , boolean useDefaultExcludes , String tempDir , TaskListener listener ) throws IOException {
this . url = url;
@@ -245,7 +222,7 @@ public void stash(String name, FilePath workspace, Launcher launcher, EnvVars en
@Override
public Integer invoke (File f , VirtualChannel channel ) throws IOException , InterruptedException {
// TODO JCLOUDS-769 streaming upload is not currently straightforward, so using a temp file pending rewrite to use multipart uploads
// TODO use streaming upload rather than a temp file; is it necessary to set the content length in advance?
// (we prefer not to upload individual files for stashes, so as to preserve symlinks & file permissions, as StashManager’s default does)
Path tempDirP = Paths . get(tempDir);
Files . createDirectories(tempDirP);
@@ -258,7 +235,7 @@ public Integer invoke(File f, VirtualChannel channel) throws IOException, Interr
throw new IOException (e);
}
if (count > 0 ) {
uploadFile(tmp, url, listener, stopAfterAttemptNumber, waitMultiplier, waitMaximum, timeout );
client . uploadFile(tmp. toFile() , url, listener);
}
return count;
} finally {
@@ -279,24 +256,29 @@ public void unstash(String name, FilePath workspace, Launcher launcher, EnvVars
String . format(" No such saved stash ‘%s’ found at %s/%s" . getContainer(), blobPath));
}
URL url = provider. toExternalURL(blob, HttpMethod . GET
workspace. act(new Unstash (url));
workspace. act(new Unstash (url, listener ));
listener. getLogger(). printf(" Unstashed file(s) from %s%n" . toURI(provider. getContainer(), blobPath));
}
private static final class Unstash extends MasterToSlaveFileCallable<Void > {
private static final long serialVersionUID = 1L ;
private final URL url;
private final TaskListener listener;
private final RobustHTTPClient client = JCloudsArtifactManager . client;
Unstash (URL url ) throws IOException {
Unstash (URL url , TaskListener listener ) throws IOException {
this . url = url;
this . listener = listener;
}
@Override
public Void invoke (File f , VirtualChannel channel ) throws IOException , InterruptedException {
try (InputStream is = url. openStream()) {
new FilePath (f). untarFrom(is, FilePath . TarCompression . GZIP
// Note that this API currently offers no count of files in the tarball we could report.
}
client. connect(" download" " download " + RobustHTTPClient . sanitize(url) + " into " + f, c - > c. execute(new HttpGet (url. toString())), response - > {
try (InputStream is = response. getEntity(). getContent()) {
new FilePath (f). untarFrom(is, FilePath . TarCompression . GZIP
// Note that this API currently offers no count of files in the tarball we could report.
}
}, listener);
return null ;
}
}
@@ -359,91 +341,4 @@ private BlobStoreContext getContext() throws IOException {
return provider. getContext();
}
private static final class HTTPAbortException extends AbortException {
final int code;
HTTPAbortException (int code , String message ) {
super (message);
this . code = code;
}
}
/**
* Number of upload attempts of nonfatal errors before giving up.
*/
static int UPLOAD_STOP_AFTER_ATTEMPT_NUMBER = Integer . getInteger(JCloudsArtifactManager . class. getName() + " .UPLOAD_STOP_AFTER_ATTEMPT_NUMBER" 10 );
/**
* Initial number of milliseconds between first and second upload attempts.
* Subsequent ones increase exponentially.
* Note that this is not a <em >randomized</em> exponential backoff;
* and the base of the exponent is hard-coded to 2.
*/
static long UPLOAD_WAIT_MULTIPLIER = Long . getLong(JCloudsArtifactManager . class. getName() + " .UPLOAD_WAIT_MULTIPLIER" 100 );
/**
* Maximum number of seconds between upload attempts.
*/
static long UPLOAD_WAIT_MAXIMUM = Long . getLong(JCloudsArtifactManager . class. getName() + " .UPLOAD_WAIT_MAXIMUM" 300 );
/**
* Number of seconds to permit a single upload attempt to take.
*/
static long UPLOAD_TIMEOUT = Long . getLong(JCloudsArtifactManager . class. getName() + " .UPLOAD_TIMEOUT" /* 15m */ 15 * 60 );
private static final ExecutorService executors = JenkinsJVM . isJenkinsJVM() ? Computer . threadPoolForRemoting : Executors . newCachedThreadPool();
/**
* Upload a file to a URL
*/
@SuppressWarnings (" Convert2Lambda" // bogus use of generics (type variable should have been on class); cannot be made into a lambda
private static void uploadFile (Path f , URL url , final TaskListener listener , int stopAfterAttemptNumber , long waitMultiplier , long waitMaximum , long timeout ) throws IOException , InterruptedException {
String urlSafe = url. toString(). replaceFirst(" [?].+$" " ?…"
try {
AtomicReference<Throwable > lastError = new AtomicReference<> ();
RetryerBuilder . < Void > newBuilder().
retryIfException(x - > x instanceof IOException && (! (x instanceof HTTPAbortException ) || ((HTTPAbortException ) x). code >= 500 ) || x instanceof UncheckedTimeoutException ).
withRetryListener(new RetryListener () {
@Override
public <Void > void onRetry (Attempt<Void > attempt ) {
if (attempt. hasException()) {
lastError. set(attempt. getExceptionCause());
}
}
}).
withStopStrategy(StopStrategies . stopAfterAttempt(stopAfterAttemptNumber)).
withWaitStrategy(WaitStrategies . exponentialWait(waitMultiplier, waitMaximum, TimeUnit . SECONDS.
withAttemptTimeLimiter(AttemptTimeLimiters . fixedTimeLimit(timeout, TimeUnit . SECONDS.
build(). call(() - > {
Throwable t = lastError. get();
if (t != null ) {
listener. getLogger(). println(" Retrying upload after: " + (t instanceof AbortException ? t. getMessage() : t. toString()));
}
HttpURLConnection connection = (HttpURLConnection ) url. openConnection();
connection. setDoOutput(true );
connection. setRequestMethod(" PUT"
connection. setFixedLengthStreamingMode(Files . size(f)); // prevent loading file in memory
try (OutputStream out = connection. getOutputStream()) {
Files . copy(f, out);
}
int responseCode = connection. getResponseCode();
if (responseCode < 200 || responseCode >= 300 ) {
String diag;
try (InputStream err = connection. getErrorStream()) {
diag = err != null ? IOUtils . toString(err, connection. getContentEncoding()) : null ;
}
throw new HTTPAbortException (responseCode, String . format(" Failed to upload %s to %s, response: %d %s, body: %s" . toAbsolutePath(), urlSafe, responseCode, connection. getResponseMessage(), diag));
}
return null ;
});
} catch (ExecutionException | RetryException x) { // *sigh*, checked exceptions
Throwable x2 = x. getCause();
if (x2 instanceof IOException ) {
throw (IOException ) x2;
} else if (x2 instanceof RuntimeException ) {
throw (RuntimeException ) x2;
} else if (x2 instanceof InterruptedException ) {
throw (InterruptedException ) x2;
} else { // Error?
throw new RuntimeException (x);
}
}
}
}
