Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[DLAB-1432] Separate computational resources #524

Merged
merged 2 commits into from Jan 13, 2020
Merged

[DLAB-1432] Separate computational resources #524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0541a08
[DLAB-1432] Separate computational resources
ofuks Jan 10, 2020
379f0b2
Changed url for creating cluster
DG1202 Jan 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 8 additions & 1 deletion ...ices/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java
View file
Open in desktop
Expand Up @@ -27,6 +27,9 @@
import com.epam.dlab.backendapi.interceptor.BudgetLimitInterceptor;
import com.epam.dlab.backendapi.resources.aws.BillingResourceAws;
import com.epam.dlab.backendapi.resources.aws.ComputationalResourceAws;
import com.epam.dlab.backendapi.resources.azure.ComputationalResourceAzure;
import com.epam.dlab.backendapi.resources.gcp.ComputationalResourceGcp;
import com.epam.dlab.backendapi.resources.gcp.GcpOauthResource;
import com.epam.dlab.backendapi.service.BillingService;
import com.epam.dlab.backendapi.service.InfrastructureInfoService;
import com.epam.dlab.backendapi.service.InfrastructureTemplateService;
Expand Down Expand Up @@ -71,10 +74,14 @@ protected void configure() {
@Override
public void init(Environment environment, Injector injector) {
environment.jersey().register(injector.getInstance(ComputationalResourceAws.class));
environment.jersey().register(injector.getInstance(ComputationalResourceAzure.class));
environment.jersey().register(injector.getInstance(ComputationalResourceGcp.class));
environment.jersey().register(injector.getInstance(BillingResourceAws.class));
if (injector.getInstance(SelfServiceApplicationConfiguration.class).isGcpOuauth2AuthenticationEnabled()) {
environment.jersey().register(injector.getInstance(GcpOauthResource.class));
}
}


@Provides
@Singleton
Scheduler provideScheduler(SelfServiceApplicationConfiguration configuration) throws SchedulerException {
Expand Down
2 changes: 1 addition & 1 deletion ...ervice/src/main/java/com/epam/dlab/backendapi/resources/aws/ComputationalResourceAws.java
View file
Open in desktop
Expand Up @@ -52,7 +52,7 @@
/**
* Provides the REST API for the computational resource on AWS.
*/
@Path("/infrastructure_provision/computational_resources")
@Path("/aws/infrastructure_provision/computational_resources")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Slf4j
Expand Down
2 changes: 1 addition & 1 deletion ...ce/src/main/java/com/epam/dlab/backendapi/resources/azure/ComputationalResourceAzure.java
View file
Open in desktop
Expand Up @@ -48,7 +48,7 @@
/**
* Provides the REST API for the computational resource on Azure.
*/
@Path("/infrastructure_provision/computational_resources")
@Path("/azure/infrastructure_provision/computational_resources")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Slf4j
Expand Down
4 changes: 4 additions & 0 deletions ...rvice/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
View file
Open in desktop
Expand Up @@ -19,11 +19,13 @@

package com.epam.dlab.backendapi.resources.dto;

import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.ToString;
import org.bson.Document;

import java.util.List;
import java.util.Map;

@AllArgsConstructor
Expand All @@ -37,4 +39,6 @@ public class ProjectInfrastructureInfo {
private Map<String, Map<String, String>> shared;
@JsonProperty
private Iterable<Document> exploratory;
@JsonProperty
private List<EndpointDTO> endpoints;
}
2 changes: 1 addition & 1 deletion ...ervice/src/main/java/com/epam/dlab/backendapi/resources/gcp/ComputationalResourceGcp.java
View file
Open in desktop
Expand Up @@ -50,7 +50,7 @@
/**
* Provides the REST API for the computational resource on GCP.
*/
@Path("/infrastructure_provision/computational_resources")
@Path("/gcp/infrastructure_provision/computational_resources")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Slf4j
Expand Down
17 changes: 12 additions & 5 deletions ...ce/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
View file
Open in desktop
Expand Up @@ -24,9 +24,11 @@
import com.epam.dlab.backendapi.dao.BillingDAO;
import com.epam.dlab.backendapi.dao.EnvDAO;
import com.epam.dlab.backendapi.dao.ExploratoryDAO;
import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
import com.epam.dlab.backendapi.resources.dto.HealthStatusPageDTO;
import com.epam.dlab.backendapi.resources.dto.ProjectInfrastructureInfo;
import com.epam.dlab.backendapi.service.EndpointService;
import com.epam.dlab.backendapi.service.InfrastructureInfoService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.dto.InfrastructureMetaInfoDTO;
Expand Down Expand Up @@ -61,28 +63,33 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService
private BillingDAO billingDAO;
@Inject
private ProjectService projectService;
@Inject
private EndpointService endpointService;


@Override
public List<ProjectInfrastructureInfo> getUserResources(String user) {
log.debug("Loading list of provisioned resources for user {}", user);
try {
Iterable<Document> documents = expDAO.findExploratory(user);

List<EndpointDTO> allEndpoints = endpointService.getEndpoints();
return StreamSupport.stream(documents.spliterator(),
false)
.collect(Collectors.groupingBy(d -> d.getString("project")))
.entrySet()
.stream()
.map(e -> {

List<ProjectEndpointDTO> endpoints = projectService.get(e.getKey()).getEndpoints();
List<EndpointDTO> endpointResult = allEndpoints.stream()
.filter(endpoint -> endpoints.stream()
.anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
.collect(Collectors.toList());
final Map<String, Map<String, String>> projectEdges =
projectService.get(e.getKey()).getEndpoints().stream()
endpoints.stream()
.collect(Collectors.toMap(ProjectEndpointDTO::getName,
endpointDTO -> getSharedInfo(endpointDTO.getEdgeInfo())));
return new ProjectInfrastructureInfo(e.getKey(),
billingDAO.getBillingProjectQuoteUsed(e.getKey()),
projectEdges, e.getValue());
billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(), endpointResult);
})
.collect(Collectors.toList());
} catch (Exception e) {
Expand Down
12 changes: 6 additions & 6 deletions ...rvice/src/main/resources/webapp/src/app/core/services/applicationServiceFacade.service.ts
View file
Open in desktop
Expand Up @@ -201,16 +201,16 @@ export class ApplicationServiceFacade {
data, { responseType: 'text', observe: 'response' });
}

public buildCreateComputationalResources_DataengineServiceRequest(data): Observable<any> {
public buildCreateComputationalResources_DataengineServiceRequest(data, provider): Observable<any> {
return this.buildRequest(HTTPMethod.PUT,
this.requestRegistry.Item(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINESERVICE),
'/api/'+ provider + this.requestRegistry.Item(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINESERVICE),
data,
{ observe: 'response' });
}

public buildCreateComputationalResources_DataengineRequest(data): Observable<any> {
public buildCreateComputationalResources_DataengineRequest(data, provider): Observable<any> {
return this.buildRequest(HTTPMethod.PUT,
this.requestRegistry.Item(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINE),
'/api/'+ provider + this.requestRegistry.Item(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINE),
data,
{ observe: 'response' });
}
Expand Down Expand Up @@ -633,9 +633,9 @@ export class ApplicationServiceFacade {
this.requestRegistry.Add(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES,
'/api/infrastructure_provision/computational_resources');
this.requestRegistry.Add(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINESERVICE,
'/api/infrastructure_provision/computational_resources/dataengine-service'); // emr(aws)
'/infrastructure_provision/computational_resources/dataengine-service'); // emr(aws)
this.requestRegistry.Add(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_DATAENGINE,
'/api/infrastructure_provision/computational_resources/dataengine'); // spark (azure|aws)
'/infrastructure_provision/computational_resources/dataengine'); // spark (azure|aws)

this.requestRegistry.Add(ApplicationServiceFacade.COMPUTATIONAL_RESOURCES_TEMLATES,
'/api/infrastructure_templates/computational_templates');
Expand Down
8 changes: 4 additions & 4 deletions ...ices/self-service/src/main/resources/webapp/src/app/core/services/userResource.service.ts
View file
Open in desktop
Expand Up @@ -82,19 +82,19 @@ export class UserResourceService {
catchError(ErrorUtils.handleServiceError));
}

public createComputationalResource_DataengineService(data): Observable<{}> {
public createComputationalResource_DataengineService(data, provider): Observable<{}> {
const body = JSON.stringify(data);
return this.applicationServiceFacade
.buildCreateComputationalResources_DataengineServiceRequest(body)
.buildCreateComputationalResources_DataengineServiceRequest(body, provider)
.pipe(
map(response => response),
catchError(ErrorUtils.handleServiceError));
}

public createComputationalResource_Dataengine(data): Observable<{}> {
public createComputationalResource_Dataengine(data, provider): Observable<{}> {
const body = JSON.stringify(data);
return this.applicationServiceFacade
.buildCreateComputationalResources_DataengineRequest(body)
.buildCreateComputationalResources_DataengineRequest(body, provider)
.pipe(
map(response => response),
catchError(ErrorUtils.handleServiceError));
Expand Down
2 changes: 1 addition & 1 deletion .../computational-resource-create-dialog/computational-resource-create-dialog.component.html
View file
Open in desktop
Expand Up @@ -194,7 +194,7 @@ <h4 class="modal-title">Add computational resources</h4>
</label>
<div class="config-link" *ngIf="(configuration?.nativeElement['checked'] || false)
&& selectedImage?.image === 'docker.dlab-dataengine-service'
&& DICTIONARY.cloud_provider === 'aws'">
&& PROVIDER === 'aws'">
To view example JSON of configurations refer for <a
href="https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html" target="_blank">AWS
official documentation</a>
Expand Down
5 changes: 2 additions & 3 deletions ...al/computational-resource-create-dialog/computational-resource-create-dialog.component.ts
View file
Open in desktop
Expand Up @@ -36,7 +36,7 @@ import { CLUSTER_CONFIGURATION } from './cluster-configuration-templates';
})

export class ComputationalResourceCreateDialogComponent implements OnInit {
readonly PROVIDER = DICTIONARY.cloud_provider;
readonly PROVIDER = this.data.notebook.cloud_provider;
readonly DICTIONARY = DICTIONARY;
readonly CLUSTER_CONFIGURATION = CLUSTER_CONFIGURATION;
readonly CheckUtils = CheckUtils;
Expand Down Expand Up @@ -76,7 +76,6 @@ export class ComputationalResourceCreateDialogComponent implements OnInit {
this.notebook_instance = this.data.notebook;
this.resourcesList = this.data.full_list;
this.initFormModel();

this.getTemplates(this.notebook_instance.project, this.notebook_instance.endpoint);
}

Expand Down Expand Up @@ -130,7 +129,7 @@ export class ComputationalResourceCreateDialogComponent implements OnInit {
}

public createComputationalResource(data) {
this.model.createComputationalResource(data, this.selectedImage, this.notebook_instance, this.spotInstance)
this.model.createComputationalResource(data, this.selectedImage, this.notebook_instance, this.spotInstance, this.PROVIDER.toLowerCase())
.subscribe((response: any) => {
if (response.status === HTTP_STATUS_CODES.OK) this.dialogRef.close();
}, error => this.toastr.error(error.message, 'Oops!'));
Expand Down
8 changes: 4 additions & 4 deletions ...computational/computational-resource-create-dialog/computational-resource-create.model.ts
View file
Open in desktop
Expand Up @@ -29,7 +29,7 @@ export class ComputationalResourceModel {

constructor(private userResourceService: UserResourceService) { }

public createComputationalResource(parameters, image, env, spot): Observable<{}> {
public createComputationalResource(parameters, image, env, spot, provider): Observable<{}> {
const config = parameters.configuration_parameters ? JSON.parse(parameters.configuration_parameters) : null;

if (DICTIONARY.cloud_provider === 'aws' && image.image === 'docker.dlab-dataengine-service') {
Expand All @@ -47,7 +47,7 @@ export class ComputationalResourceModel {
config: config,
project: env.project,
custom_tag: parameters.custom_tag
});
}, provider);
} else if (DICTIONARY.cloud_provider === 'gcp' && image.image === 'docker.dlab-dataengine-service') {
return this.userResourceService.createComputationalResource_DataengineService({
template_name: image.template_name,
Expand All @@ -63,7 +63,7 @@ export class ComputationalResourceModel {
config: config,
project: env.project,
custom_tag: parameters.custom_tag
});
}, provider);
} else {
return this.userResourceService.createComputationalResource_Dataengine({
name: parameters.cluster_alias_name,
Expand All @@ -75,7 +75,7 @@ export class ComputationalResourceModel {
config: config,
project: env.project,
custom_tag: parameters.custom_tag
});
}, provider);
}
}
}
2 changes: 1 addition & 1 deletion .../src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
View file
Open in desktop
Expand Up @@ -195,7 +195,7 @@
</li>
</div>
<li (click)="exploratoryAction(element, 'ami')"
*ngIf="element.status === 'running' && DICTIONARY.cloud_provider !== 'gcp' && element.image !== 'docker.dlab-superset' && element.image !== 'docker.dlab-jupyterlab'">
*ngIf="element.status === 'running' && element.cloud_provider !== 'gcp' && element.image !== 'docker.dlab-superset' && element.image !== 'docker.dlab-jupyterlab'">
<i class="material-icons">view_module</i>
<span>Create {{ DICTIONARY.image }}</span>
</li>
Expand Down
4 changes: 3 additions & 1 deletion ...ervice/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
View file
Open in desktop
Expand Up @@ -49,6 +49,7 @@ export class ExploratoryModel {
public project: string,
public endpoint: string,
public tags: any,
public cloud_provider: string
) { }

public static loadEnvironments(data: Array<any>) {
Expand Down Expand Up @@ -82,7 +83,8 @@ export class ExploratoryModel {
value.shared[el.endpoint][DICTIONARY.datalake_shared_directory_name],
el.project,
el.endpoint,
el.tags
el.tags,
value.endpoints[0].cloudProvider.toLowerCase()
))
}
});
Expand Down