srm-client: Port SRM client to Apache HTTP components and CANL

Motivation: The SRM client uses the old Axis 1.4 (due to the use of old SOAP encodings). Axis 1.4 by default uses an internal simplistic HTTP client without support for keeping connections alive between calls. Axis 1.4 ships with support for the Apache Commons HTTP client 3.x, but that version is deprecated. The SRM client uses JGlobus to obtain GSI support for Axis. We want to get rid of JGlobus in dCache. Modification: Introduces the HttpClientSender and HttpClientTransport, allowing Axis 1.4 to use the HTTP client from Apache Commons HTTP Components. The code is based on the original code in Axis, but heavily updated to make as much use of the features of the HTTP client as possible. Introduces the GsiHttpClientSender, which subclasses HttpClientSender to add a JGlobus independent client GSI layer. Makes use of CANL for certificate handling and proxy generation. Updates our client tools to use the new transport. Result: Faster startup for users with large /etc/grid-security/certificates, faster calls to the SRM server if more than one call is made (in particular for high latency links). For localhost and a short list of CA certs I have observed slightly higher startup cost - probably due to the extra JARs. May change once we get rid of JGlobus completely. In particular the srmfs and delegation shells are much more responsive. Proper RFC 2818 host name verification. This means that the host name provided in the URI MUST also be the host name in a subject alternative name (SAN) of the common name of the server certificate. No reverse lookup is done and no Globus wildcards are support (but RFC 2818 compliant wildcards are). This means this client should not be deployed by sites or users relying on the old insecure behaviour. Within EMI, sites are required to switch to proper use of SAN at the end of 2015. We should continue to distribute the old client for this purpose. Target: trunk Require-notes: yes Require-book: no Acked-by: Paul Millar <paul.millar@desy.de> Patch: https://rb.dcache.org/r/8613/
dCache · Oct 16, 2015 · 22ab9db · 22ab9db
1 parent 3780943
commit 22ab9db
Show file tree

Hide file tree

Showing 43 changed files with 1,811 additions and 736 deletions.
diff --git a/modules/srm-client/pom.xml b/modules/srm-client/pom.xml
@@ -34,10 +34,6 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.jglobus</groupId>
-            <artifactId>axisg</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.dcache</groupId>
             <artifactId>srm-common</artifactId>

diff --git a/modules/srm-client/src/main/bin/delegation b/modules/srm-client/src/main/bin/delegation
@@ -11,11 +11,18 @@ else
     logbackDefn=-Dlogback.configurationFile=$conf_dir/logback.xml
 fi
 
+if [ -n "$X509_USER_PROXY" ]; then
+    x509_user_proxy="$X509_USER_PROXY"
+else
+    x509_user_proxy=/tmp/x509up_u$(id -u)
+fi
+
 CLASSPATH="$jar_dir/*" java -Dlog=${DELEGATION_LOG:-warn} \
     -client \
     -Djava.awt.headless=true \
     -DwantLog4jSetup=n \
     $logbackDefn \
     -XX:+TieredCompilation \
     -XX:TieredStopAtLevel=0 \
-    org.dcache.srm.DelegationShell "$@"
+    ${SRM_JAVA_OPTIONS} \
+    org.dcache.srm.DelegationShell -x509_user_proxy="$x509_user_proxy" "$@"
diff --git a/modules/srm-client/src/main/bin/srmfs b/modules/srm-client/src/main/bin/srmfs
@@ -11,11 +11,25 @@ else
     logbackDefn=-Dlogback.configurationFile=$conf_dir/logback.xml
 fi
 
+if [ -n "$X509_USER_PROXY" ]; then
+    x509_user_proxy="$X509_USER_PROXY"
+elif [ -r /tmp/x509up_u$(id -u) ]; then
+    x509_user_proxy=/tmp/x509up_u$(id -u)
+fi
+
+if [ -n "$X509_CERT_DIR" ]; then
+   x509_user_trusted_certs=$X509_CERT_DIR
+elif [ -d "$HOME/.globus/certificates" ]; then
+   x509_user_trusted_certs=$HOME/.globus/certificates
+else
+   x509_user_trusted_certs=/etc/grid-security/certificates
+fi
+
 CLASSPATH="$jar_dir/*" java -Dlog=${DELEGATION_LOG:-warn} \
     -client \
     -Djava.awt.headless=true \
     -DwantLog4jSetup=n \
     $logbackDefn \
     -XX:+TieredCompilation \
     -XX:TieredStopAtLevel=1 \
-    org.dcache.srm.shell.SrmShell "$@"
+    org.dcache.srm.shell.SrmShell -x509_user_proxy="$x509_user_proxy" -x509_user_trusted_certificates="$x509_user_trusted_certs" "$@"
diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMAbortFilesClientV2.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMAbortFilesClientV2.java
@@ -9,11 +9,11 @@
 
 package gov.fnal.srm.util;
 
+import eu.emi.security.authn.x509.X509Credential;
 import org.apache.axis.types.URI;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
 
 import java.io.IOException;
+import java.util.Date;
 
 import org.dcache.srm.client.SRMClientV2;
 import org.dcache.srm.util.RequestStatusTool;
@@ -26,36 +26,31 @@
 
 public class SRMAbortFilesClientV2 extends SRMClient {
     private ISRM isrm;
-    private GSSCredential credential;
+    private X509Credential credential;
 
     public SRMAbortFilesClientV2(Configuration configuration) {
         super(configuration);
     }
 
     @Override
     public void connect() throws Exception {
-        credential=getGssCredential();
+        credential= getCredential();
         isrm = new SRMClientV2(configuration.getSrmUrl(),
-                credential,
-                configuration.getRetry_timeout(),
-                configuration.getRetry_num(),
-                doDelegation,
-                fullDelegation,
-                gss_expected_name,
-                configuration.getWebservice_path(),
-                configuration.getTransport());
+                               credential,
+                               configuration.getRetry_timeout(),
+                               configuration.getRetry_num(),
+                               doDelegation,
+                               fullDelegation,
+                               gss_expected_name,
+                               configuration.getWebservice_path(),
+                               configuration.getX509_user_trusted_certificates(),
+                               configuration.getTransport());
     }
 
     @Override
     public void start() throws Exception{
-        try {
-            if (credential.getRemainingLifetime() < 60) {
-                throw new Exception(
-                        "Remaining lifetime of credential is less than a minute.");
-            }
-        }
-        catch (GSSException gsse) {
-            throw gsse;
+        if (credential.getCertificate().getNotAfter().before(new Date())) {
+            throw new RuntimeException("credentials have expired");
         }
         StringBuilder sb = new StringBuilder();
         boolean failed=false;

diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMAbortRequestClientV2.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMAbortRequestClientV2.java
@@ -9,11 +9,11 @@
 
 package gov.fnal.srm.util;
 
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
+import eu.emi.security.authn.x509.X509Credential;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.Date;
 
 import org.dcache.srm.client.SRMClientV2;
 import org.dcache.srm.util.RequestStatusTool;
@@ -26,7 +26,7 @@
 public class SRMAbortRequestClientV2 extends SRMClient {
     private ISRM isrm;
     private URI srmURL;
-    private GSSCredential credential;
+    private X509Credential credential;
 
     public SRMAbortRequestClientV2(Configuration configuration,
                                    URI url) {
@@ -36,28 +36,23 @@ public SRMAbortRequestClientV2(Configuration configuration,
 
     @Override
     public void connect() throws Exception {
-        credential=getGssCredential();
+        credential= getCredential();
         isrm = new SRMClientV2(srmURL,
-                credential,
-                configuration.getRetry_timeout(),
-                configuration.getRetry_num(),
-                doDelegation,
-                fullDelegation,
-                gss_expected_name,
-                configuration.getWebservice_path(),
-                configuration.getTransport());
+                               credential,
+                               configuration.getRetry_timeout(),
+                               configuration.getRetry_num(),
+                               doDelegation,
+                               fullDelegation,
+                               gss_expected_name,
+                               configuration.getWebservice_path(),
+                               configuration.getX509_user_trusted_certificates(),
+                               configuration.getTransport());
     }
 
     @Override
     public void start() throws Exception{
-        try {
-            if (credential.getRemainingLifetime() < 60) {
-                throw new Exception(
-                        "Remaining lifetime of credential is less than a minute.");
-            }
-        }
-        catch (GSSException gsse) {
-            throw gsse;
+        if (credential.getCertificate().getNotAfter().before(new Date())) {
+            throw new RuntimeException("credentials have expired");
         }
         for (String requestToken : configuration.getArrayOfRequestTokens()) {
             try {

diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMBringOnlineClientV2.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMBringOnlineClientV2.java
@@ -128,14 +128,15 @@ public SRMBringOnlineClientV2(Configuration configuration, java.net.URI[] from)
     public void connect() throws Exception {
         java.net.URI srmUrl = from[0];
         srmv2 = new SRMClientV2(srmUrl,
-                getGssCredential(),
-                configuration.getRetry_timeout(),
-                configuration.getRetry_num(),
-                doDelegation,
-                fullDelegation,
-                gss_expected_name,
-                configuration.getWebservice_path(),
-                configuration.getTransport());
+                                getCredential(),
+                                configuration.getRetry_timeout(),
+                                configuration.getRetry_num(),
+                                doDelegation,
+                                fullDelegation,
+                                gss_expected_name,
+                                configuration.getWebservice_path(),
+                                configuration.getX509_user_trusted_certificates(),
+                                configuration.getTransport());
     }
 
     public void setProtocols(String[] protocols) {

diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMCheckPermissionClientV2.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMCheckPermissionClientV2.java
@@ -72,9 +72,10 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 
 package gov.fnal.srm.util;
 
+import eu.emi.security.authn.x509.X509Credential;
 import org.apache.axis.types.URI;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
+
+import java.util.Date;
 
 import org.dcache.srm.client.SRMClientV2;
 import org.dcache.srm.v2_2.ArrayOfAnyURI;
@@ -88,7 +89,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import org.dcache.srm.v2_2.TStatusCode;
 
 public class SRMCheckPermissionClientV2 extends SRMClient {
-    private GSSCredential cred;
+    private X509Credential cred;
     private java.net.URI[] surls;
     private String[] surl_string;
     private ISRM isrm;
@@ -99,7 +100,7 @@ public SRMCheckPermissionClientV2(Configuration configuration,
         this.surls       = surls;
         this.surl_string = surl_string;
         try {
-            cred = getGssCredential();
+            cred = getCredential();
         }
         catch (Exception e) {
             cred = null;
@@ -111,26 +112,21 @@ public SRMCheckPermissionClientV2(Configuration configuration,
     public void connect() throws Exception {
         java.net.URI srmUrl = surls[0];
         isrm = new SRMClientV2(srmUrl,
-                getGssCredential(),
-                configuration.getRetry_timeout(),
-                configuration.getRetry_num(),
-                doDelegation,
-                fullDelegation,
-                gss_expected_name,
-                configuration.getWebservice_path(),
-                configuration.getTransport());
+                               getCredential(),
+                               configuration.getRetry_timeout(),
+                               configuration.getRetry_num(),
+                               doDelegation,
+                               fullDelegation,
+                               gss_expected_name,
+                               configuration.getWebservice_path(),
+                               configuration.getX509_user_trusted_certificates(),
+                               configuration.getTransport());
     }
 
     @Override
     public void start() throws Exception {
-        try {
-            if (cred.getRemainingLifetime() < 60) {
-                throw new Exception(
-                        "Remaining lifetime of credential is less than a minute.");
-            }
-        }
-        catch (GSSException gsse) {
-            throw gsse;
+        if (cred.getCertificate().getNotAfter().before(new Date())) {
+            throw new RuntimeException("credentials have expired");
         }
         ArrayOfAnyURI surlarray=new ArrayOfAnyURI();
         URI[] uriarray=new URI[surl_string.length];

diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMClient.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMClient.java
@@ -67,6 +67,9 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 
 package gov.fnal.srm.util;
 
+import eu.emi.security.authn.x509.X509Credential;
+import eu.emi.security.authn.x509.impl.KeyAndCertCredential;
+import eu.emi.security.authn.x509.impl.PEMCredential;
 import org.ietf.jgss.GSSCredential;
 
 import java.io.IOException;
@@ -174,7 +177,7 @@ protected void connect(java.net.URI srmUrl) throws Exception {
         try {
 
             SRMClientV1 client;
-            client = new SRMClientV1(srmUrl, getGssCredential(),
+            client = new SRMClientV1(srmUrl, getCredential(),
                     configuration.getRetry_timeout(),
                     configuration.getRetry_num(),
                     doDelegation, fullDelegation, gss_expected_name,
@@ -194,19 +197,11 @@ protected void connect(java.net.URI srmUrl) throws Exception {
 
     }
 
-    public GSSCredential  getGssCredential()
-    throws Exception {
-        if(configuration.isUseproxy()) {
-            return SslGsiSocketFactory.
-            createUserCredential(configuration.getX509_user_proxy(),
-                    null,
-                    null);
+    public X509Credential getCredential() throws Exception {
+        if (configuration.isUseproxy()) {
+            return new PEMCredential(configuration.getX509_user_proxy(), (char[]) null);
         } else {
-            return SslGsiSocketFactory.
-            createUserCredential(
-                    null,
-                    configuration.getX509_user_cert(),
-                    configuration.getX509_user_key());
+            return new PEMCredential(configuration.getX509_user_key(), configuration.getX509_user_cert(), null);
         }
     }
 

diff --git a/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMCopyClientV2.java b/modules/srm-client/src/main/java/gov/fnal/srm/util/SRMCopyClientV2.java
@@ -100,11 +100,11 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 
 package gov.fnal.srm.util;
 
+import eu.emi.security.authn.x509.X509Credential;
 import org.apache.axis.types.URI;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
 
 import java.io.IOException;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -141,7 +141,7 @@ public class SRMCopyClientV2 extends SRMClient implements Runnable {
     private java.net.URI to[];
     private SrmCopyRequest req = new SrmCopyRequest();
 
-    private GSSCredential cred;
+    private X509Credential cred;
     private ISRM srmv2;
     private Thread hook;
     private HashMap<java.net.URI,Integer> pendingSurlsMap = new HashMap<>();
@@ -152,7 +152,7 @@ public SRMCopyClientV2(Configuration configuration, java.net.URI[] from, java.ne
         this.from = from;
         this.to = to;
         try {
-            cred = getGssCredential();
+            cred = getCredential();
         } catch (Exception e) {
             cred = null;
             System.err.println("Couldn't getGssCredential.");
@@ -168,25 +168,21 @@ public void connect() throws Exception {
             srmUrl = to[0];
         }
         srmv2 = new SRMClientV2(srmUrl,
-                getGssCredential(),
-                configuration.getRetry_timeout(),
-                configuration.getRetry_num(),
-                doDelegation,
-                fullDelegation,
-                gss_expected_name,
-                configuration.getWebservice_path(),
-                configuration.getTransport());
+                                getCredential(),
+                                configuration.getRetry_timeout(),
+                                configuration.getRetry_num(),
+                                doDelegation,
+                                fullDelegation,
+                                gss_expected_name,
+                                configuration.getWebservice_path(),
+                                configuration.getX509_user_trusted_certificates(),
+                                configuration.getTransport());
     }
 
     @Override
     public void start() throws Exception {
-        try {
-            if (cred.getRemainingLifetime() < 60) {
-                throw new Exception(
-                        "Remaining lifetime of credential is less than a minute.");
-            }
-        } catch (GSSException gsse) {
-            throw gsse;
+        if (cred.getCertificate().getNotAfter().before(new Date())) {
+            throw new RuntimeException("credentials have expired");
         }
         try {
             //