Merge pull request #159 from integratedmodelling/IM-331-Retrieve-Open…

…EO-credentials-from-endpoint

Im 331 retrieve open eo credentials from endpoint

adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/openeo/OpenEO.java

@@ -31,6 +31,7 @@
 import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor;
 import org.integratedmodelling.klab.auth.Authorization;
 import org.integratedmodelling.klab.exceptions.KlabIOException;
+import org.integratedmodelling.klab.exceptions.KlabMissingCredentialsException;
 import org.integratedmodelling.klab.exceptions.KlabRemoteException;
 import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials;
 import org.integratedmodelling.klab.rest.Notification;
@@ -395,11 +396,12 @@ public void encodeSelf(String url) {
 	}
 
 	public OpenEO(String endpoint) {
-		this.endpoint = endpoint;
 		ExternalAuthenticationCredentials credentials = Authentication.INSTANCE.getCredentials(endpoint);
-		if (credentials != null) {
-			this.authorization = new Authorization(credentials);
+		if (credentials == null) {
+			throw new KlabMissingCredentialsException(endpoint);
 		}
+		this.endpoint = credentials.getURL();
+		this.authorization = new Authorization(credentials, endpoint);
 		this.executor.scheduleAtFixedRate(() -> {
 			Set<Job> finished = new HashSet<>();
 			for (Job job : jobs) {

api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/rest/ExternalAuthenticationCredentials.java

@@ -15,7 +15,7 @@ public class ExternalAuthenticationCredentials {
     static {
         parameterKeys = new HashMap<>();
         parameterKeys.put("basic", new String[]{"username", "password"});
-        parameterKeys.put("oidc", new String[]{"url", "grant_type", "client_id", "client_secrets", "scope", "provider_id"});
+        parameterKeys.put("oidc", new String[]{"grant_type", "client_id", "client_secrets", "provider_id"});
         parameterKeys.put("s3", new String[]{"accessKey", "secretKey"});
         parameterKeys.put("key", new String[]{"key"});
     }
@@ -24,7 +24,7 @@ public class ExternalAuthenticationCredentials {
      * Credentials, depending on scheme
      * 
      * for basic: username and password
-     * for oidc: Authentication URL, grant type, client ID, client secret, scope, provider
+     * for oidc: grant type, client ID, client secret, provider
      * for s3: endpoint URL, access key, secret key
      * for key: a single key
      */
@@ -35,6 +35,11 @@ public class ExternalAuthenticationCredentials {
      */
     private String scheme = "basic";
 
+    /**
+     * The URL of the authentication service
+     */
+    private String url = "";
+
     public List<String> getCredentials() {
         return credentials;
     }
@@ -51,4 +56,11 @@ public void setScheme(String scheme) {
         this.scheme = scheme;
     }
 
+    public String getURL() {
+        return url;
+    }
+
+    public void setURL(String url) {
+        this.url = url;
+    }
 }

klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java

@@ -2,12 +2,15 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.security.Principal;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -433,8 +436,35 @@ public IUserIdentity reauthenticate() {
       return authenticate(this.certificate);
     }
 
-    public ExternalAuthenticationCredentials getCredentials(String hostUrl) {
-        return externalCredentials.get(hostUrl);
+    /**
+     * Returns a credential for the selected host. It can be an ID or an URL as long as it is presented in the credentials.
+     * @param endpoint as a URL or an ID defined in the credentials
+     * @return
+     */
+    public ExternalAuthenticationCredentials getCredentials(String endpoint) {
+        if (endpoint.startsWith("http://") || endpoint.startsWith("https://")) {
+            return getCredentialsByUrl(endpoint);
+        }
+        return getCredentialsById(endpoint);
+    }
+
+    /**
+     * Returns a credential for the selected host URL. If multiple host URL are present, the first in alphabetical order will be returned.
+     * @param hostUrl
+     * @return the credential or null if not present
+     */
+    private ExternalAuthenticationCredentials getCredentialsByUrl(String hostUrl) {
+        return externalCredentials.values().stream().filter(credential -> credential.getURL().equals(hostUrl))
+                .sorted().findFirst().orElse(null);
+    }
+
+    /**
+     * Returns a credential for the selected ID.
+     * @param id
+     * @return the credential or null if not present
+     */
+    private ExternalAuthenticationCredentials getCredentialsById(String id) {
+        return externalCredentials.get(id);
     }
 
     /**
@@ -515,8 +545,50 @@ public Collection<ISession> getSessions() {
         return ret;
     }
 
-    public void addExternalCredentials(String host, ExternalAuthenticationCredentials credentials) {
-        externalCredentials.put(host, credentials);
+    /**
+     * This method has been created so legacy credential scheme is updated into the new credentials scheme if possible.
+     * The main changes are:
+     * - Use of a unique ID instead of the url as key
+     * - Move the URL as its own parameter
+     * - Changes on the scheme of oidc credentials
+     */
+    public void updateLegacyCredentials() {
+        boolean isLegacyDetected = externalCredentials.values().stream()
+                .anyMatch(credential -> credential.getURL() == null || credential.getURL().isEmpty());
+        if (!isLegacyDetected) {
+            return;
+        }
+        Set<String> legacyCredentialIds = new HashSet<String>();
+        externalCredentials.keySet().parallelStream().forEach(id -> {
+            if (externalCredentials.get(id).getURL() != null && !externalCredentials.get(id).getURL().isEmpty()) {
+                return;
+            }
+            if (!(id.startsWith("http://") || id.startsWith("https://"))) {
+                legacyCredentialIds.add(id);
+                return;
+            }
+            ExternalAuthenticationCredentials credentials = externalCredentials.get(id);
+            credentials.setURL(id);
+            // Former scheme of oidc: "url", "grant_type", "client_id", "client_secrets", "scope", "provider_id"
+            if (credentials.getScheme().equals("oidc")) {
+                credentials.getCredentials().remove(4); // Former scope
+                credentials.getCredentials().remove(0); // Former url
+            }
+            try {
+                String hostname = new URL(id).getHost();
+                externalCredentials.put(hostname, credentials);
+                legacyCredentialIds.add(id);
+            } catch (MalformedURLException e) {
+                // Do nothing, this credential will be removed
+            }
+        });
+        // We remove the old credentials by their IDs
+        legacyCredentialIds.forEach(id -> externalCredentials.remove(id));
+        externalCredentials.write();
+    }
+
+    public void addExternalCredentials(String id, ExternalAuthenticationCredentials credentials) {
+        externalCredentials.put(id, credentials);
         externalCredentials.write();
     }
 

klab.engine/src/main/java/org/integratedmodelling/klab/auth/Authorization.java

@@ -1,12 +1,15 @@
 package org.integratedmodelling.klab.auth;
 
 import java.util.Base64;
+import java.util.List;
+import java.util.stream.Collectors;
 
 import org.integratedmodelling.klab.Authentication;
 import org.integratedmodelling.klab.exceptions.KlabAuthorizationException;
 import org.integratedmodelling.klab.exceptions.KlabIOException;
 import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException;
 import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials;
+import org.integratedmodelling.klab.utils.Pair;
 
 import kong.unirest.HttpResponse;
 import kong.unirest.JsonNode;
@@ -38,7 +41,16 @@ public class Authorization {
      * @param credentials
      */
     public Authorization(ExternalAuthenticationCredentials credentials) {
+        this(credentials, null);
+    }
 
+    /**
+     * Create a new authorization. {@link #isOnline()} should be called after creation.
+     * 
+     * @param credentials
+     * @param endpoint
+     */
+    public Authorization(ExternalAuthenticationCredentials credentials, String endpoint) {
         if (credentials == null) {
             throw new KlabIllegalArgumentException("attempted authorization with null credentials");
         }
@@ -51,6 +63,9 @@ public Authorization(ExternalAuthenticationCredentials credentials) {
                     .encode((credentials.getCredentials().get(0) + ":" + credentials.getCredentials().get(1)).getBytes());
             this.token = new String(encodedBytes);
         } else if ("oidc".equals(credentials.getScheme())) {
+            if (endpoint == null) {
+                throw new KlabIllegalArgumentException("Attempted to start an OIDC authoritation workflow without an endpoint");
+            }
             refreshToken();
         }
     }
@@ -64,18 +79,45 @@ public boolean isOnline() {
         return token != null;
     }
 
+    private Pair<String, String> parseProvider() {
+        HttpResponse<JsonNode> response = Unirest.get(credentials.getURL() + "/credentials/oidc").asJson();
+        if (!response.isSuccess()) {
+            throw new KlabAuthorizationException("Cannot access " + credentials.getURL() + " for OIDC authentication");
+        }
+        List<JSONObject> providers = response.getBody().getObject().getJSONArray("providers").toList();
+        JSONObject provider = providers.stream().filter(prov -> prov.getString("id").equals(credentials.getCredentials().get(3))).findFirst()
+                .orElseThrow(() -> new KlabAuthorizationException("No known provider '" + credentials.getCredentials().get(3) + "' at " + credentials.getURL()));
+        List<String> scopes = provider.getJSONArray("scopes").toList();
+        String scope = scopes.stream().collect(Collectors.joining(" "));
+        return new Pair<>(provider.getString("issuer"), scope);
+    }
+
+    private String parseIssuer(String issuerUrl) {
+        HttpResponse<JsonNode> response = Unirest.get(issuerUrl + "/.well-known/openid-configuration").asJson();
+        if (!response.isSuccess()) {
+            throw new KlabAuthorizationException("Cannot access " + issuerUrl + " for OIDC authentication");
+        }
+        return response.getBody().getObject().getString("token_endpoint");
+    }
+
     /**
      * OIDC-style token
      */
     private void refreshToken() {
-
         /*
-         * authenticate and get the first token. Credentials should contain: 0. Auth endpoint 1.
-         * grant type 2. client ID 3. client secret 4. scope 5. provider
+         * authenticate and get the first token. Credentials should contain:
+         * 0. grant type 1. client ID 2. client secret 3. provider
          */
-        MultipartBody query = Unirest.post(credentials.getCredentials().get(0))
-                .field("grant_type", credentials.getCredentials().get(1)).field("client_id", credentials.getCredentials().get(2))
-                .field("client_secret", credentials.getCredentials().get(3)).field("scope", credentials.getCredentials().get(4));
+        Pair<String, String> issuerAndScope = parseProvider();
+        String issuer = issuerAndScope.getFirst();
+        String scope = issuerAndScope.getSecond();
+        String tokenServiceUrl = parseIssuer(issuer);
+
+        MultipartBody query = Unirest.post(tokenServiceUrl)
+                .field("grant_type", credentials.getCredentials().get(0))
+                .field("client_id", credentials.getCredentials().get(1))
+                .field("client_secret", credentials.getCredentials().get(2))
+                .field("scope", scope);
 
         if (this.token != null) {
             query = query.header("Authorization:",
@@ -101,7 +143,7 @@ private void refreshToken() {
             if (token != null && duration < 0) {
                 duration = response.has("expires_in") ? response.getLong("expires_in") : 0;
             }
-            this.prefix = "oidc/" + credentials.getCredentials().get(5) + "/";
+            this.prefix = "oidc/" + credentials.getCredentials().get(3) + "/";
             this.expiry += (duration * 1000l);
         }
     }
@@ -142,9 +184,10 @@ public String getAuthorization() {
     }
 
     public static void main(String[] args) throws InterruptedException {
-        ExternalAuthenticationCredentials crds = Authentication.INSTANCE.getCredentials("https://openeo.vito.be/openeo/1.1.0");
+        String endpoint = "https://openeo.vito.be/openeo/1.1.0";
+        ExternalAuthenticationCredentials crds = Authentication.INSTANCE.getCredentials(endpoint);
         if (crds != null) {
-            Authorization authorization = new Authorization(crds);
+            Authorization authorization = new Authorization(crds, endpoint);
             System.out.println(authorization.getAuthorization());
             System.out.println("Sleeping 300 seconds: don't change the channel");
             Thread.sleep(300000l);

klab.engine/src/main/java/org/integratedmodelling/klab/cli/commands/credentials/Set.java

@@ -1,5 +1,7 @@
 package org.integratedmodelling.klab.cli.commands.credentials;
 
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.List;
 
 import org.integratedmodelling.kim.api.IServiceCall;
@@ -19,26 +21,47 @@ public class Set implements ICommand {
 
     @Override
     public Object execute(IServiceCall call, ISession session) {
-
         String ret = "";
+        if (call.getParameters().containsKey("update")) {
+            Authentication.INSTANCE.updateLegacyCredentials();
+            ret += "Credentials updated.";
+        }
+
         List<?> args = (List<?>) call.getParameters().get("arguments");
-        String scheme = "basic";
-        int nargs = 3;
+        if (args.isEmpty()) {
+            return ret;
+        }
+
+        String id = (String) call.getParameters().get("id");
+        String url = (String) call.getParameters().get("url");
+        String scheme = (String) call.getParameters().get("scheme");
         int astart = 0;
-        if (args.size() > 0) {
-            if (ExternalAuthenticationCredentials.parameterKeys.containsKey(args.get(0).toString())) {
-                scheme = args.get(0).toString();
-                astart = 1;
+
+        if (scheme == null) {
+            scheme = args.get(astart).toString();
+            astart++;
+        }
+        List<String> params = List.of(ExternalAuthenticationCredentials.parameterKeys.get(scheme));
+        int nargs = params.size() + astart;
+
+        if (url == null) {
+            url = args.get(astart).toString();
+            astart++;
+            nargs++;
+        }
+        if (id == null) {
+            try {
+                id = new URL(url).getHost();
+            } catch (MalformedURLException e) {
+               throw new KlabValidationException("Cannot generate credential ID from URL " + url);
             }
         }
 
-        String[] params = ExternalAuthenticationCredentials.parameterKeys.get(scheme);
-        if (params == null) {
+        if (params.isEmpty()) {
             throw new KlabIllegalArgumentException("unrecognized authorization scheme");
         }
 
-        nargs = params.length + 1; // the URL
-        if (args.size() != (nargs + astart)) {
+        if (args.size() != nargs) {
             throw new KlabValidationException("expecting " + nargs + " arguments for scheme " + scheme);
         }
 
@@ -47,13 +70,13 @@ public Object execute(IServiceCall call, ISession session) {
         ExternalAuthenticationCredentials credentials = new ExternalAuthenticationCredentials();
 
         credentials.setScheme(scheme);
-        for (int i = astart + 1; i < (astart + nargs); i++) {
+        credentials.setURL(url);
+        for (int i = astart; i < nargs; i++) {
             credentials.getCredentials().add(args.get(i).toString());
-            ret += "\n   " + params[i - astart - 1] + ": " + args.get(i).toString();
-
-        }
+            ret += "\n   " + params.get(i - astart) + ": " + args.get(i).toString();
+        };
 
-        Authentication.INSTANCE.addExternalCredentials(args.get(astart).toString(), credentials);
+        Authentication.INSTANCE.addExternalCredentials(id, credentials);
 
         return ret;