Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce PureModelContextCollection type and use it in WorkspaceSDLCLoader #2641

Closed
wants to merge 12 commits into from
Closed

Introduce PureModelContextCollection type and use it in WorkspaceSDLCLoader #2641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
2eeddb1
add failing test for sdlc load dependency issue
abhishoya-gs Feb 15, 2024
bb862c7
Fetch all dependencies PMCD using new Metadata API
abhishoya-gs Mar 12, 2024
68a0d74
revert change for gitlab profile in identity factory
abhishoya-gs Mar 12, 2024
cf883d8
fix checkstyle
abhishoya-gs Mar 12, 2024
539d07b
introduce PureModelContextCollection
abhishoya-gs Mar 13, 2024
0f94233
fixes
abhishoya-gs Mar 13, 2024
f5c1bf5
test fixes
abhishoya-gs Mar 13, 2024
ec704d3
fix checkstyle
abhishoya-gs Mar 13, 2024
ce5d7d7
fix TestSDLCLoader
abhishoya-gs Mar 14, 2024
bc92f61
fix TestGraphQLApiAbstract
abhishoya-gs Mar 14, 2024
082f08a
remove constraints in SDLC Loader, change caching
abhishoya-gs Mar 14, 2024
f68b9ae
add missing import
abhishoya-gs Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 0 additions & 7 deletions ...se/legend-engine-core-language-pure/legend-engine-language-pure-modelManager-sdlc/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,6 @@
</dependency>
<!-- ENGINE -->

<!-- PAC4J -->
<dependency>
<groupId>org.pac4j</groupId>
<artifactId>pac4j-core</artifactId>
</dependency>
<!-- PAC4J -->

<!-- JACKSON -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
23 changes: 23 additions & 0 deletions ...lc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,22 @@
import io.opentracing.Scope;
import io.opentracing.Span;
import io.opentracing.util.GlobalTracer;

import java.util.Collection;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.security.auth.Subject;
import org.apache.http.impl.client.CloseableHttpClient;
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.modelManager.sdlc.alloy.AlloySDLCLoader;
import org.finos.legend.engine.language.pure.modelManager.sdlc.pure.PureServerLoader;
import org.finos.legend.engine.language.pure.modelManager.sdlc.workspace.WorkspaceSDLCLoader;
import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureSDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.SDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.SDLCVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.context.WorkspaceSDLC;
import org.finos.legend.engine.shared.core.identity.Identity;
Expand Down Expand Up @@ -78,6 +83,24 @@ public PureModelContextData visit(AlloySDLC sdlc)
}
}

@Override
public PureModelContextData visit(Collection<SDLC> sdlcCollection)
{
parentSpan.setTag("sdlc", "collection");
try (Scope ignore = GlobalTracer.get().buildSpan("Request Alloy Metadata").startActive(true))
{
if (sdlcCollection.isEmpty())
{
return PureModelContextData.newPureModelContextData();
}
else if (sdlcCollection.stream().allMatch(sdlc -> sdlc instanceof AlloySDLC))
{
return this.alloyLoader.loadAlloyProjects(identity, sdlcCollection.stream().map(sdlc1 -> (AlloySDLC) sdlc1).collect(Collectors.toList()), clientVersion, this.httpClientProvider);
}
throw new RuntimeException("Invalid collection of SDLC");
}
}

@Override
public PureModelContextData visit(PureSDLC pureSDLC)
{
Expand Down
70 changes: 57 additions & 13 deletions ...dlc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
import io.opentracing.Span;
import io.opentracing.tag.Tags;
import io.opentracing.util.GlobalTracer;

import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.security.auth.Subject;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
Expand All @@ -43,6 +47,7 @@
import org.finos.legend.engine.language.pure.modelManager.sdlc.workspace.WorkspaceSDLCLoader;
import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextCollection;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureSDLC;
Expand Down Expand Up @@ -128,7 +133,25 @@ public void setModelManager(ModelManager modelManager)
@Override
public boolean shouldCache(PureModelContext context)
{
return this.supports(context) && (isCacheablePureSDLC(((PureModelContextPointer) context).sdlcInfo) || isCacheableAlloySDLC(((PureModelContextPointer) context).sdlcInfo));
return this.supports(context) &&
(
(
context instanceof PureModelContextCollection &&
isCacheableCollection((PureModelContextCollection) context)
) ||
(
context instanceof PureModelContextPointer &&
(
isCacheablePureSDLC(((PureModelContextPointer) context).sdlcInfo) ||
isCacheableAlloySDLC(((PureModelContextPointer) context).sdlcInfo)
)
)
);
}

private boolean isCacheableCollection(PureModelContextCollection context)
{
return !context.getContexts().isEmpty() && context.getContexts().stream().allMatch(this::shouldCache);
}

private boolean isCacheablePureSDLC(SDLC sdlc)
Expand All @@ -148,30 +171,36 @@ private boolean isCacheableAlloySDLC(SDLC sdlc)
@Override
public PureModelContext cacheKey(PureModelContext context, Identity identity)
{
if (isCacheablePureSDLC(((PureModelContextPointer) context).sdlcInfo))
if (context instanceof PureModelContextPointer)
{
final Subject executionSubject = getSubject();
Function0<PureModelContext> pureModelContextFunction = () -> this.pureLoader.getCacheKey(context, identity, executionSubject);
return executionSubject == null ? pureModelContextFunction.value() : exec(executionSubject, pureModelContextFunction::value);
if (isCacheablePureSDLC(((PureModelContextPointer) context).sdlcInfo))
{
final Subject executionSubject = getSubject();
Function0<PureModelContext> pureModelContextFunction = () -> this.pureLoader.getCacheKey(context, identity, executionSubject);
return executionSubject == null ? pureModelContextFunction.value() : exec(executionSubject, pureModelContextFunction::value);
}
else
{
return this.alloyLoader.getCacheKey(context);
}
}
else
else if (context instanceof PureModelContextCollection)
{
return this.alloyLoader.getCacheKey(context);
List<PureModelContext> contextsCacheKeys = ((PureModelContextCollection) context).getContexts().stream().map(pureModelContext -> this.cacheKey(pureModelContext, identity)).collect(Collectors.toList());
return new PureModelContextCollection(contextsCacheKeys);
}
throw new RuntimeException("Invalid PureModelContext type for generating cache key");
}

@Override
public boolean supports(PureModelContext context)
{
return context instanceof PureModelContextPointer;
return context instanceof PureModelContextPointer || context instanceof PureModelContextCollection;
}

@Override
public PureModelContextData load(Identity identity, PureModelContext ctx, String clientVersion, Span parentSpan)
{
PureModelContextPointer context = (PureModelContextPointer) ctx;
Assert.assertTrue(clientVersion != null, () -> "Client version should be set when pulling metadata from the metadata repository");

SDLCFetcher fetcher = new SDLCFetcher(
parentSpan,
clientVersion,
Expand All @@ -181,9 +210,24 @@ public PureModelContextData load(Identity identity, PureModelContext ctx, String
this.alloyLoader,
this.workspaceLoader
);

Subject subject = getSubject();
PureModelContextData metaData = subject == null ? context.sdlcInfo.accept(fetcher) : exec(subject, () -> context.sdlcInfo.accept(fetcher));
PureModelContextData metaData;
Assert.assertTrue(clientVersion != null, () -> "Client version should be set when pulling metadata from the metadata repository");

if (ctx instanceof PureModelContextPointer)
{
PureModelContextPointer context = (PureModelContextPointer) ctx;
metaData = subject == null ? context.sdlcInfo.accept(fetcher) : exec(subject, () -> context.sdlcInfo.accept(fetcher));
}
else if (ctx instanceof PureModelContextCollection)
{
Collection<SDLC> sdlcList = ((PureModelContextCollection) ctx).getContexts().stream().map(context1 -> ((PureModelContextPointer) context1).sdlcInfo).collect(Collectors.toList());
metaData = subject == null ? fetcher.visit(sdlcList) : exec(subject, () -> fetcher.visit(sdlcList));
}
else
{
throw new RuntimeException("Invalid call to SDLCLoader.load");
}

if (metaData.origin != null)
{
Expand Down
70 changes: 67 additions & 3 deletions ...n/java/org/finos/legend/engine/language/pure/modelManager/sdlc/alloy/AlloySDLCLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,92 @@

package org.finos.legend.engine.language.pure.modelManager.sdlc.alloy;

import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicHeader;
import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader;
import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.MetaDataServerConfiguration;
import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.finos.legend.engine.shared.core.identity.Identity;
import org.finos.legend.engine.shared.core.operational.Assert;
import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;

import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

import static javax.ws.rs.core.HttpHeaders.ACCEPT;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE;

public class AlloySDLCLoader
{
private final MetaDataServerConfiguration metaDataServerConfiguration;
private final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapper();

public AlloySDLCLoader(MetaDataServerConfiguration metaDataServerConfiguration)
{
this.metaDataServerConfiguration = metaDataServerConfiguration;
}

public class ProjectVersion
{
private final String groupId;
private final String artifactId;
private final String versionId;

public ProjectVersion(String groupId, String artifactId, String versionId)
{
this.groupId = groupId;
this.artifactId = artifactId;
this.versionId = versionId;
}

public String getGroupId()
{
return groupId;
}

public String getArtifactId()
{
return artifactId;
}

public String getVersionId()
{
return versionId;
}
}

public PureModelContextData loadAlloyProject(Identity identity, AlloySDLC alloySDLC, String clientVersion, Function<Identity, CloseableHttpClient> httpClientProvider)
{
return SDLCLoader.loadMetadataFromHTTPURL(identity, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, getMetaDataApiUrl(identity, alloySDLC, clientVersion), httpClientProvider);
}

public PureModelContextData loadAlloyProjects(Identity identity, List<AlloySDLC> alloySDLC, String clientVersion, Function<Identity, CloseableHttpClient> httpClientProvider)
{
return SDLCLoader.loadMetadataFromHTTPURL(identity, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, getMetaDataApiUrl(identity, alloySDLC, clientVersion), httpClientProvider, (url) ->
{
try
{
HttpPost httpPost = new HttpPost(url);
List<ProjectVersion> projectVersions = alloySDLC.stream().map(sdlc -> new ProjectVersion(sdlc.groupId, sdlc.artifactId, sdlc.version)).collect(Collectors.toList());
httpPost.setEntity(new StringEntity(objectMapper.writeValueAsString(projectVersions)));
httpPost.setHeader(new BasicHeader(CONTENT_TYPE, APPLICATION_JSON));
return httpPost;
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}

public String getMetaDataApiUrl(Identity identity, AlloySDLC alloySDLC, String clientVersion)
{
Assert.assertTrue(alloySDLC.project == null, () -> "Accessing metadata services using project id was demised. Please update AlloySDLC to provide group and artifact IDs");
Expand All @@ -50,6 +109,11 @@ public String getMetaDataApiUrl(Identity identity, AlloySDLC alloySDLC, String c
metaDataServerConfiguration.getAlloy().getBaseUrl() + "/projects/" + alloySDLC.groupId + "/" + alloySDLC.artifactId + "/versions/" + alloySDLC.version + "/pureModelContextData?clientVersion=" + clientVersion;
}

public String getMetaDataApiUrl(Identity identity, List<AlloySDLC> alloySDLC, String clientVersion)
{
return metaDataServerConfiguration.getAlloy().getBaseUrl() + "/projects/dependencies/pureModelContextData?clientVersion=" + clientVersion;
}

public List<String> checkAllPathsExist(PureModelContextData data, AlloySDLC alloySDLC)
{
List<String> pathsFromPointer = alloySDLC.packageableElementPointers.stream().map(s -> s.path).collect(Collectors.toList());
Expand Down
1 change: 1 addition & 0 deletions ...e/pure/modelManager/sdlc/configuration/MetadataServerPrivateAccessTokenConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@
public class MetadataServerPrivateAccessTokenConfiguration extends MetadataServerPac4jConfiguration
{
public String accessTokenHeaderName;
public String clientName;
}
48 changes: 25 additions & 23 deletions ...rg/finos/legend/engine/language/pure/modelManager/sdlc/workspace/WorkspaceSDLCLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,22 +23,27 @@
import java.security.PrivilegedAction;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.security.auth.Subject;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicHeader;
import org.finos.legend.engine.language.pure.modelManager.ModelManager;
import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader;
import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.MetadataServerPrivateAccessTokenConfiguration;
import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.ServerConnectionConfiguration;
import org.finos.legend.engine.protocol.Protocol;
import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextCollection;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer;
import org.finos.legend.engine.protocol.pure.v1.model.context.WorkspaceSDLC;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.finos.legend.engine.shared.core.identity.Identity;
import org.finos.legend.engine.shared.core.identity.credential.OAuthCredential;
import org.finos.legend.engine.shared.core.identity.factory.*;
import org.finos.legend.engine.shared.core.kerberos.HttpClientBuilder;
import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
Expand Down Expand Up @@ -105,16 +110,7 @@ public PureModelContextData getSDLCDependenciesPMCD(Identity identity, String cl
try (InputStream content = entity.getContent())
{
List<SDLCProjectDependency> dependencies = mapper.readValue(content, SDLC_PROJECT_DEPENDENCY_TYPE);

PureModelContextData.Builder builder = PureModelContextData.newBuilder();

dependencies.forEach(dependency ->
{
builder.addPureModelContextData(this.loadDependencyData(identity, clientVersion, dependency));
});

builder.removeDuplicates();
return builder.build();
return this.loadDependencyData(identity, clientVersion, dependencies);
}
}
catch (Exception e)
Expand All @@ -137,11 +133,12 @@ private HttpGet prepareHttpRequest(Identity identity, String url)
if (this.sdlcServerConnectionConfig.pac4j != null && this.sdlcServerConnectionConfig.pac4j instanceof MetadataServerPrivateAccessTokenConfiguration)
{
String patHeaderName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConnectionConfig.pac4j).accessTokenHeaderName;
//MutableList<GitlabPersonalAccessTokenProfile> patProfiles = pm.selectInstancesOf(GitlabPersonalAccessTokenProfile.class);
if (identity != null)
String clientName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConnectionConfig.pac4j).clientName;
if (identity != null && identity.getCredential(OAuthCredential.class).isPresent())
{
httpRequest = new HttpGet(String.format("%s?client_name=%s", url, IdentityFactoryProvider.getInstance().adapt(identity).get(0).getClientName()));
//httpRequest.addHeader(new BasicHeader(patHeaderName, patProfiles.getFirst().getPersonalAccessToken()));
String accessToken = identity.getCredential(OAuthCredential.class).get().getAccessToken();
httpRequest = new HttpGet(String.format("%s?client_name=%s", url, clientName));
httpRequest.addHeader(new BasicHeader(patHeaderName, accessToken));
}
}

Expand All @@ -153,16 +150,21 @@ private HttpGet prepareHttpRequest(Identity identity, String url)
return httpRequest;
}

private PureModelContextData loadDependencyData(Identity profiles, String clientVersion, SDLCProjectDependency dependency)
private PureModelContextData loadDependencyData(Identity profiles, String clientVersion, List<SDLCProjectDependency> dependencies)
{
PureModelContextPointer pointer = new PureModelContextPointer();
AlloySDLC sdlcInfo = new AlloySDLC();
sdlcInfo.groupId = dependency.getGroupId();
sdlcInfo.artifactId = dependency.getArtifactId();
sdlcInfo.version = dependency.getVersionId();
pointer.sdlcInfo = sdlcInfo;
pointer.serializer = new Protocol("pure", clientVersion);
return this.modelManager.loadData(pointer, clientVersion, profiles);
List<PureModelContext> contexts = dependencies.stream().map(dependency ->
{
PureModelContextPointer pointer = new PureModelContextPointer();
AlloySDLC sdlcInfo = new AlloySDLC();
sdlcInfo.groupId = dependency.getGroupId();
sdlcInfo.artifactId = dependency.getArtifactId();
sdlcInfo.version = dependency.getVersionId();
pointer.sdlcInfo = sdlcInfo;
pointer.serializer = new Protocol("pure", clientVersion);
return pointer;
}
).collect(Collectors.toList());
return this.modelManager.loadData(new PureModelContextCollection(contexts), clientVersion, profiles);
}

private static class SDLCProjectDependency
Expand Down