Fetch all dependencies PMCD using new Metadata API

Support GitlabPersonalAccessTokenClient
finos · Mar 12, 2024 · e2292c6 · e2292c6
1 parent a232516
commit e2292c6
Show file tree

Hide file tree

Showing 9 changed files with 169 additions and 27 deletions.
diff --git a/...lc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCFetcher.java b/...lc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCFetcher.java
@@ -22,6 +22,7 @@
 import java.util.function.Function;
 import javax.security.auth.Subject;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.eclipse.collections.impl.factory.Lists;
 import org.eclipse.collections.impl.utility.ListIterate;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.alloy.AlloySDLCLoader;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.pure.PureServerLoader;
@@ -78,6 +79,18 @@ public PureModelContextData visit(AlloySDLC sdlc)
         }
     }
 
+    @Override
+    public PureModelContextData visit(List<AlloySDLC> sdlc)
+    {
+        parentSpan.setTag("sdlc", "alloy");
+        try (Scope ignore = GlobalTracer.get().buildSpan("Request Alloy Metadata").startActive(true))
+        {
+            PureModelContextData loadedProject = this.alloyLoader.loadAlloyProjects(identity, sdlc, clientVersion, this.httpClientProvider);
+            return loadedProject;
+            // TODO add check for missing paths
+        }
+    }
+
     @Override
     public PureModelContextData visit(PureSDLC pureSDLC)
     {

diff --git a/...dlc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCLoader.java b/...dlc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCLoader.java
@@ -20,9 +20,11 @@
 import io.opentracing.tag.Tags;
 import io.opentracing.util.GlobalTracer;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import javax.security.auth.Subject;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
@@ -195,6 +197,35 @@ public PureModelContextData load(Identity identity, PureModelContext ctx, String
         return metaData;
     }
 
+    @Override
+    public PureModelContextData load(Identity identity, List<PureModelContext> contextList, String clientVersion, Span parentSpan)
+    {
+        List<AlloySDLC> sdlcList = contextList.stream().map(context1 -> (AlloySDLC) ((PureModelContextPointer) context1).sdlcInfo).collect(Collectors.toList());
+        Assert.assertTrue(clientVersion != null, () -> "Client version should be set when pulling metadata from the metadata repository");
+
+        SDLCFetcher fetcher = new SDLCFetcher(
+                parentSpan,
+                clientVersion,
+                this.httpClientProvider,
+                identity,
+                this.pureLoader,
+                this.alloyLoader,
+                this.workspaceLoader
+        );
+
+        Subject subject = getSubject();
+        PureModelContextData metaData = subject == null ? fetcher.visit(sdlcList) : exec(subject, () -> fetcher.visit(sdlcList));
+
+        if (metaData.origin != null)
+        {
+            Assert.assertTrue("none".equals(metaData.origin.sdlcInfo.version), () -> "Version can't be set in the pointer");
+            metaData.origin.sdlcInfo.version = metaData.origin.sdlcInfo.baseVersion;
+            metaData.origin.sdlcInfo.baseVersion = null;
+        }
+
+        return metaData;
+    }
+
     public static PureModelContextData loadMetadataFromHTTPURL(Identity identity, LoggingEventType startEvent, LoggingEventType stopEvent, String url)
     {
         return loadMetadataFromHTTPURL(identity, startEvent, stopEvent, url, null);

diff --git a/...n/java/org/finos/legend/engine/language/pure/modelManager/sdlc/alloy/AlloySDLCLoader.java b/...n/java/org/finos/legend/engine/language/pure/modelManager/sdlc/alloy/AlloySDLCLoader.java
@@ -14,33 +14,85 @@
 
 package org.finos.legend.engine.language.pure.modelManager.sdlc.alloy;
 
-import java.util.List;
-import java.util.function.Function;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.MetaDataServerConfiguration;
 import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.engine.shared.core.identity.Identity;
 import org.finos.legend.engine.shared.core.operational.Assert;
 import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
 
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static javax.ws.rs.core.HttpHeaders.ACCEPT;
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;
+
 public class AlloySDLCLoader
 {
     private final MetaDataServerConfiguration metaDataServerConfiguration;
+    private final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapper();
 
     public AlloySDLCLoader(MetaDataServerConfiguration metaDataServerConfiguration)
     {
         this.metaDataServerConfiguration = metaDataServerConfiguration;
     }
 
+    public class ProjectVersion
+    {
+        private final String groupId;
+        private final String artifactId;
+        private final String versionId;
+
+        public ProjectVersion(String groupId, String artifactId, String versionId) {
+            this.groupId = groupId;
+            this.artifactId = artifactId;
+            this.versionId = versionId;
+        }
+
+        public String getGroupId() {
+            return groupId;
+        }
+
+        public String getArtifactId() {
+            return artifactId;
+        }
+
+        public String getVersionId() {
+            return versionId;
+        }
+    }
+
     public PureModelContextData loadAlloyProject(Identity identity, AlloySDLC alloySDLC, String clientVersion, Function<Identity, CloseableHttpClient> httpClientProvider)
     {
         return SDLCLoader.loadMetadataFromHTTPURL(identity, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, getMetaDataApiUrl(identity, alloySDLC, clientVersion), httpClientProvider);
     }
 
+    public PureModelContextData loadAlloyProjects(Identity identity, List<AlloySDLC> alloySDLC, String clientVersion, Function<Identity, CloseableHttpClient> httpClientProvider)
+    {
+        return SDLCLoader.loadMetadataFromHTTPURL(identity, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, getMetaDataApiUrl(identity, alloySDLC, clientVersion), httpClientProvider, (url) ->
+        {
+            try {
+                HttpPost httpPost = new HttpPost(url);
+                List<ProjectVersion> projectVersions = alloySDLC.stream().map(sdlc -> new ProjectVersion(sdlc.groupId, sdlc.artifactId, sdlc.version)).collect(Collectors.toList());
+                httpPost.setEntity(new StringEntity(objectMapper.writeValueAsString(projectVersions)));
+                httpPost.setHeader(new BasicHeader(CONTENT_TYPE, APPLICATION_JSON));
+                return httpPost;
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
     public String getMetaDataApiUrl(Identity identity, AlloySDLC alloySDLC, String clientVersion)
     {
         Assert.assertTrue(alloySDLC.project == null, () -> "Accessing metadata services using project id was demised.  Please update AlloySDLC to provide group and artifact IDs");
@@ -50,6 +102,11 @@ public String getMetaDataApiUrl(Identity identity, AlloySDLC alloySDLC, String c
                 metaDataServerConfiguration.getAlloy().getBaseUrl() + "/projects/" + alloySDLC.groupId + "/" + alloySDLC.artifactId + "/versions/" + alloySDLC.version + "/pureModelContextData?clientVersion=" + clientVersion;
     }
 
+    public String getMetaDataApiUrl(Identity identity, List<AlloySDLC> alloySDLC, String clientVersion)
+    {
+        return metaDataServerConfiguration.getAlloy().getBaseUrl() + "/projects/dependencies/pureModelContextData?clientVersion=" + clientVersion;
+    }
+
     public List<String> checkAllPathsExist(PureModelContextData data, AlloySDLC alloySDLC)
     {
         List<String> pathsFromPointer = alloySDLC.packageableElementPointers.stream().map(s -> s.path).collect(Collectors.toList());

diff --git a/...rg/finos/legend/engine/language/pure/modelManager/sdlc/workspace/WorkspaceSDLCLoader.java b/...rg/finos/legend/engine/language/pure/modelManager/sdlc/workspace/WorkspaceSDLCLoader.java
@@ -23,6 +23,7 @@
 import java.security.PrivilegedAction;
 import java.util.List;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import javax.security.auth.Subject;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.HttpGet;
@@ -35,11 +36,13 @@
 import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.ServerConnectionConfiguration;
 import org.finos.legend.engine.protocol.Protocol;
 import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer;
 import org.finos.legend.engine.protocol.pure.v1.model.context.WorkspaceSDLC;
 import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.engine.shared.core.identity.Identity;
+import org.finos.legend.engine.shared.core.identity.credential.OAuthCredential;
 import org.finos.legend.engine.shared.core.identity.factory.*;
 import org.finos.legend.engine.shared.core.kerberos.HttpClientBuilder;
 import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
@@ -106,16 +109,7 @@ public PureModelContextData getSDLCDependenciesPMCD(Identity identity, String cl
                 try (InputStream content = entity.getContent())
                 {
                     List<SDLCProjectDependency> dependencies = mapper.readValue(content, SDLC_PROJECT_DEPENDENCY_TYPE);
-
-                    PureModelContextData.Builder builder = PureModelContextData.newBuilder();
-
-                    dependencies.forEach(dependency ->
-                    {
-                        builder.addPureModelContextData(this.loadDependencyData(identity, clientVersion, dependency));
-                    });
-
-                    builder.removeDuplicates();
-                    return builder.build();
+                    return this.loadDependencyData(identity, clientVersion, dependencies);
                 }
             }
             catch (Exception e)
@@ -138,11 +132,12 @@ private HttpGet prepareHttpRequest(Identity identity, String url)
         if (this.sdlcServerConnectionConfig.pac4j != null && this.sdlcServerConnectionConfig.pac4j instanceof MetadataServerPrivateAccessTokenConfiguration)
         {
             String patHeaderName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConnectionConfig.pac4j).accessTokenHeaderName;
-            //MutableList<GitlabPersonalAccessTokenProfile> patProfiles = pm.selectInstancesOf(GitlabPersonalAccessTokenProfile.class);
-            if (identity != null)
+            String clientName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConnectionConfig.pac4j).clientName;
+            if(identity != null && identity.getCredential(OAuthCredential.class).isPresent())
             {
-                httpRequest = new HttpGet(String.format("%s?client_name=%s", url, IdentityFactoryProvider.getInstance().adapt(identity).get(0).getClientName()));
-                //httpRequest.addHeader(new BasicHeader(patHeaderName, patProfiles.getFirst().getPersonalAccessToken()));
+                String accessToken = identity.getCredential(OAuthCredential.class).get().getAccessToken();
+                httpRequest = new HttpGet(String.format("%s?client_name=%s", url, clientName)); // TODO client name for sdlc should also come from config and should match with sdlc config
+                httpRequest.addHeader(new BasicHeader(patHeaderName, accessToken));
             }
         }
 
@@ -154,16 +149,19 @@ private HttpGet prepareHttpRequest(Identity identity, String url)
         return httpRequest;
     }
 
-    private PureModelContextData loadDependencyData(Identity profiles, String clientVersion, SDLCProjectDependency dependency)
+    private PureModelContextData loadDependencyData(Identity profiles, String clientVersion, List<SDLCProjectDependency> dependencies)
     {
-        PureModelContextPointer pointer = new PureModelContextPointer();
-        AlloySDLC sdlcInfo = new AlloySDLC();
-        sdlcInfo.groupId = dependency.getGroupId();
-        sdlcInfo.artifactId = dependency.getArtifactId();
-        sdlcInfo.version = dependency.getVersionId();
-        pointer.sdlcInfo = sdlcInfo;
-        pointer.serializer = new Protocol("pure", clientVersion);
-        return this.modelManager.loadData(pointer, clientVersion, profiles);
+        List<PureModelContext> pointers = dependencies.stream().map(dependency -> {
+            PureModelContextPointer pointer = new PureModelContextPointer();
+            AlloySDLC sdlcInfo = new AlloySDLC();
+            sdlcInfo.groupId = dependency.getGroupId();
+            sdlcInfo.artifactId = dependency.getArtifactId();
+            sdlcInfo.version = dependency.getVersionId();
+            pointer.sdlcInfo = sdlcInfo;
+            pointer.serializer = new Protocol("pure", clientVersion);
+            return pointer;
+        }).collect(Collectors.toList());
+        return this.modelManager.loadData(pointers, clientVersion, profiles);
     }
 
     private static class SDLCProjectDependency

diff --git a/...Manager/src/main/java/org/finos/legend/engine/language/pure/modelManager/ModelLoader.java b/...Manager/src/main/java/org/finos/legend/engine/language/pure/modelManager/ModelLoader.java
@@ -19,12 +19,16 @@
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.shared.core.identity.Identity;
 
+import java.util.List;
+
 public interface ModelLoader
 {
     boolean supports(PureModelContext context);
 
     PureModelContextData load(Identity identity, PureModelContext context, String clientVersion, Span parentSpan);
 
+    PureModelContextData load(Identity identity, List<PureModelContext> contextList, String clientVersion, Span parentSpan);
+
     void setModelManager(ModelManager modelManager);
 
     // Caching

diff --git a/...anager/src/main/java/org/finos/legend/engine/language/pure/modelManager/ModelManager.java b/...anager/src/main/java/org/finos/legend/engine/language/pure/modelManager/ModelManager.java
@@ -20,6 +20,8 @@
 import io.opentracing.Scope;
 import io.opentracing.Tracer;
 import io.opentracing.util.GlobalTracer;
+
+import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import org.eclipse.collections.api.block.procedure.Procedure;
@@ -32,6 +34,7 @@
 import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextText;
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda;
 import org.finos.legend.engine.shared.core.ObjectMapperFactory;
@@ -129,6 +132,23 @@ else if (context instanceof PureModelContextText)
         }
     }
 
+    public PureModelContextData loadData(List<PureModelContext> pureModelContextList, String clientVersion, Identity identity)
+    {
+        try (Scope scope = tracer.buildSpan("Load Model").startActive(true))
+        {
+            scope.span().setTag("context", pureModelContextList.get(0).getClass().getSimpleName());
+            if (pureModelContextList.stream().allMatch(pureModelContext -> pureModelContext instanceof PureModelContextPointer))
+            {
+                ModelLoader loader = this.modelLoaderForContext(pureModelContextList.get(0));
+                return loader.load(identity, pureModelContextList, clientVersion, scope.span());
+            }
+            else
+            {
+                throw new UnsupportedOperationException("Invalid arguments - PureModelContextList should have all elements of type PureModelContextPointer");
+            }
+        }
+    }
+
     public ModelLoader modelLoaderForContext(PureModelContext context)
     {
         MutableList<ModelLoader> loaders = modelLoaders.select(loader -> loader.supports(context));

diff --git a/...ure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/context/SDLCVisitor.java b/...ure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/context/SDLCVisitor.java
@@ -15,11 +15,15 @@
 
 package org.finos.legend.engine.protocol.pure.v1.model.context;
 
+import java.util.List;
+
 public interface SDLCVisitor<T>
 {
     T visit(AlloySDLC alloySDLC);
 
     T visit(PureSDLC pureSDLC);
 
     T visit(WorkspaceSDLC workspaceSDLC);
+
+    T visit(List<AlloySDLC> alloySDLCList);
 }
diff --git a/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml b/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml
@@ -178,6 +178,10 @@
             <artifactId>commons-io</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.shared</groupId>
+            <artifactId>legend-shared-pac4j-gitlab</artifactId>
+        </dependency>
         <!-- TEST -->
     </dependencies>
 </project>
diff --git a/...ain/java/org/finos/legend/engine/shared/core/identity/factory/DefaultIdentityFactory.java b/...ain/java/org/finos/legend/engine/shared/core/identity/factory/DefaultIdentityFactory.java
@@ -21,8 +21,10 @@
 import org.finos.legend.engine.shared.core.identity.Credential;
 import org.finos.legend.engine.shared.core.identity.Identity;
 import org.finos.legend.engine.shared.core.identity.credential.LegendKerberosCredential;
+import org.finos.legend.engine.shared.core.identity.credential.OAuthCredential;
 import org.finos.legend.engine.shared.core.kerberos.SubjectTools;
 import org.finos.legend.server.pac4j.kerberos.KerberosProfile;
+import org.finos.legend.server.pac4j.gitlab.GitlabPersonalAccessTokenProfile;
 import org.pac4j.core.profile.CommonProfile;
 
 import javax.security.auth.Subject;
@@ -69,7 +71,11 @@ public Identity makeIdentity(MutableList<CommonProfile> profiles)
         {
             return INSTANCE.makeIdentity(kerberosProfileHolder.get().getSubject());
         }
-
+        Optional<GitlabPersonalAccessTokenProfile> gitlabPersonalAccessTokenProfileHolder = this.getGitlabProfile(profiles);
+        if (gitlabPersonalAccessTokenProfileHolder.isPresent())
+        {
+            return new Identity(gitlabPersonalAccessTokenProfileHolder.get().getUsername(), new OAuthCredential(gitlabPersonalAccessTokenProfileHolder.get().getPersonalAccessToken()));
+        }
         return INSTANCE.makeUnknownIdentity();
     }
 
@@ -78,6 +84,11 @@ private Optional<KerberosProfile> getKerberosProfile(MutableList<CommonProfile>
         return Optional.ofNullable(LazyIterate.selectInstancesOf(profiles, KerberosProfile.class).getFirst());
     }
 
+    private Optional<GitlabPersonalAccessTokenProfile> getGitlabProfile(MutableList<CommonProfile> profiles)
+    {
+        return Optional.ofNullable(LazyIterate.selectInstancesOf(profiles, GitlabPersonalAccessTokenProfile.class).getFirst());
+    }
+
     public Identity makeUnknownIdentity()
     {
         return new Identity("_UNKNOWN_");