Skip to content

Commit

Permalink
add eviction to pipeline settings, make get_environment_capabilities …
Browse files Browse the repository at this point in the history 
…more robust to errors
  • Loading branch information
@ricklamers
ricklamers committed Nov 12, 2020
1 parent f4a0b08 commit 95bc2aa
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 127 deletions.
16 changes: 9 additions & 7 deletions lib/python/orchest-internals/_orchest/internals/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import uuid
import requests
import urllib.parse
import logging

from docker.types import DeviceRequest, Mount

Expand Down Expand Up @@ -69,17 +70,18 @@ def get_environment_capabilities(environment_uuid, project_uuid):
"http://orchest-webserver/store/environments/%s/%s"
% (project_uuid, environment_uuid)
)

response.raise_for_status()

except Exception as e:
print(e)
logging.error(
"Failed to get environment for environment_uuid[%s] and project_uuid[%s]. Error: %s (%s)"
% (environment_uuid, project_uuid, e, type(e))
)
return capabilities

else:
environment = response.json()
environment = response.json()

if environment["gpu_support"] == True:
capabilities += ["gpu", "utility", "compute"]
if environment["gpu_support"] == True:
capabilities += ["gpu", "utility", "compute"]

return capabilities

Expand Down
4 changes: 2 additions & 2 deletions services/memory-server/app/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ def construct_pipeline(pipeline_fname):
description = json.load(f)

try:
auto_eviction = description["settings"].get("auto-eviction", False)
auto_eviction = description["settings"].get("auto_eviction", False)
except KeyError:
auto_eviction = False

Expand Down Expand Up @@ -94,7 +94,7 @@ def start_manager(store_socket_name, pipeline_fname):
# (aka edge) is set to 1. If the outdegree of a step is equal to the
# sum of the weight of its outgoing edges, then we know that all the
# receiving steps have already read the output. If the
# `auto-eviction` is set in the `pipeline.json`, then this will
# `auto_eviction` is set in the `pipeline.json`, then this will
# cause that output to be removed from the store.
pipeline = construct_pipeline(pipeline_fname=pipeline_fname)

Expand Down
2 changes: 1 addition & 1 deletion services/memory-server/tests/pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "my-pipeline",
"uuid": "pipeline-uuid",
"settings": {
"auto-eviction": true
"auto_eviction": true
},
"steps": {
"uuid-1______________": {
Expand Down
32 changes: 3 additions & 29 deletions services/orchest-webserver/app/app/views/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,7 @@ def pipelines_create(project_uuid):
"name": request.json["name"],
"version": "1.0.0",
"uuid": pipeline_uuid,
"settings": {"auto_eviction": False},
"steps": {},
}

Expand All @@ -713,33 +714,6 @@ def pipelines_create(project_uuid):
409,
)

# Note: only pipelines in project directories can be renamed (not in experiments)
@app.route(
"/async/pipelines/rename/<project_uuid>/<pipeline_uuid>", methods=["POST"]
)
def pipelines_rename(project_uuid, pipeline_uuid):

if Pipeline.query.filter(Pipeline.uuid == pipeline_uuid).count() > 0:

pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid)

if os.path.isfile(pipeline_json_path):

with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)

pipeline_json["name"] = request.form.get("name")

with open(pipeline_json_path, "w") as json_file:
json_file.write(json.dumps(pipeline_json, indent=2))

json_string = json.dumps({"success": True})
return json_string, 200, {"content-type": "application/json"}
else:
return "", 404
else:
return "", 404

class ImportGitProjectListResource(Resource):
def post(self):
n_uuid = str(uuid.uuid4())
Expand Down Expand Up @@ -1044,7 +1018,7 @@ def notebook_html_get(project_uuid, pipeline_uuid, step_uuid):
@app.route(
"/async/pipelines/json/<project_uuid>/<pipeline_uuid>", methods=["GET", "POST"]
)
def pipelines_json_get(project_uuid, pipeline_uuid):
def pipelines_json(project_uuid, pipeline_uuid):

pipeline_json_path = get_pipeline_path(
pipeline_uuid,
Expand Down Expand Up @@ -1078,7 +1052,7 @@ def pipelines_json_get(project_uuid, pipeline_uuid):
with open(pipeline_json_path, "w") as json_file:
json_file.write(json.dumps(pipeline_json, indent=2))

return jsonify({"success": True})
return jsonify({"message": "Successfully saved pipeline."})

elif request.method == "GET":

Expand Down
162 changes: 74 additions & 88 deletions services/orchest-webserver/app/static/js/views/PipelineSettingsView.js
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,28 @@
import React from "react";
import PipelineView from "./PipelineView";
import { MDCTextField } from "@material/textfield";
import {
makeRequest,
PromiseManager,
makeCancelable,
RefManager,
} from "../lib/utils/all";
import MDCButtonReact from "../lib/mdc-components/MDCButtonReact";
import MDCTabBarReact from "../lib/mdc-components/MDCTabBarReact";
import MDCCheckboxReact from "../lib/mdc-components/MDCCheckboxReact";
import MDCTextFieldReact from "../lib/mdc-components/MDCTextFieldReact";
import MDCLinearProgressReact from "../lib/mdc-components/MDCLinearProgressReact";

class PipelineSettingsView extends React.Component {
constructor(props) {
super(props);

this.state = {
active_tab_index: 0,
restartingMemoryServer: false,
};

this.promiseManager = new PromiseManager();
this.refManager = new RefManager();
}

initComponent() {
this.initiateMDCComponents();
}

componentWillUnmount() {
this.promiseManager.cancelCancelablePromises();
}
Expand All @@ -46,28 +42,14 @@ class PipelineSettingsView extends React.Component {
if (result.success) {
let pipelineJson = JSON.parse(result["pipeline_json"]);
this.setState({ pipelineJson: pipelineJson });

this.initComponent();
} else {
console.warn("Could not load pipeline.json");
console.log(result);
}
});
}

initiateMDCComponents() {
if (this.refManager.refs.pipelineNameField) {
this.pipelineNameField = new MDCTextField(
this.refManager.refs.pipelineNameField
);
this.pipelineNameField.value = this.state.pipelineJson.name;
this.pipelineNameField.focus();
}
}

componentDidUpdate(prevProps, prevState, snapshot) {
this.initiateMDCComponents();
}
componentDidUpdate(prevProps, prevState, snapshot) {}

closeSettings() {
orchest.loadView(PipelineView, {
Expand All @@ -76,30 +58,39 @@ class PipelineSettingsView extends React.Component {
});
}

onChangeName(value) {
this.state.pipelineJson.name = value;
}
onChangeEviction(value) {
// create settings object if it doesn't exist
if (!this.state.pipelineJson.settings) {
this.state.pipelineJson.settings = {};
}

this.state.pipelineJson.settings.auto_eviction = value == true;
}

saveGeneralForm(e) {
e.preventDefault();

// new name
let pipelineName = this.pipelineNameField.value;

let formData = new FormData();
formData.append("name", pipelineName);
formData.append("pipeline_json", JSON.stringify(this.state.pipelineJson));

// perform POST to save
makeRequest(
"POST",
`/async/pipelines/rename/${this.props.project_uuid}/${this.props.pipeline_uuid}`,
`/async/pipelines/json/${this.props.project_uuid}/${this.props.pipeline_uuid}`,
{ type: "FormData", content: formData }
).then((response) => {
let json = JSON.parse(response);
console.log(json);
if (json.success === true) {
)
.then(() => {
orchest.loadView(PipelineView, {
pipeline_uuid: this.props.pipeline_uuid,
project_uuid: this.props.project_uuid,
});
}
});
})
.catch((response) => {
console.error(response);
});
}

restartMemoryServer() {
Expand All @@ -118,7 +109,7 @@ class PipelineSettingsView extends React.Component {
);

restartPromise.promise
.then((response) => {
.then(() => {
this.setState({
restartingMemoryServer: false,
});
Expand Down Expand Up @@ -149,65 +140,60 @@ class PipelineSettingsView extends React.Component {
<div className={"view-page view-pipeline-settings"}>
<h2>Pipeline settings</h2>

<MDCTabBarReact items={["General"]} icons={["subject"]} />

<div className={"tab-content"}>
{(() => {
switch (this.state.active_tab_index) {
case 0:
return (
{(() => {
if (this.state.pipelineJson) {
return (
<div>
<form>
<div>
<form>
<div>
<div
ref={this.refManager.nrefs.pipelineNameField}
className="mdc-text-field"
>
<input
type="text"
id="my-text-field"
onChange={this.stub}
className="mdc-text-field__input"
/>
<label
className="mdc-floating-label"
htmlFor="my-text-field"
>
Pipeline name
</label>
<div className="mdc-line-ripple"></div>
</div>
</div>

<MDCButtonReact
label="save"
classNames={["mdc-button--raised"]}
onClick={this.saveGeneralForm.bind(this)}
/>
</form>

<h3 className="push-up push-down">Data passing</h3>

<MDCButtonReact
disabled={this.state.restartingMemoryServer}
label="Clear memory"
icon="memory"
classNames={["mdc-button--raised"]}
onClick={this.restartMemoryServer.bind(this)}
<MDCTextFieldReact
ref={this.refManager.nrefs.pipelineNameTextField}
value={this.state.pipelineJson.name}
onChange={this.onChangeName.bind(this)}
label="Pipeline name"
classNames={["push-down"]}
/>

{(() => {
if (this.state.restartingMemoryServer) {
return (
<p className="push-up">Restarting in progress...</p>
);
<MDCCheckboxReact
value={
this.state.pipelineJson.settings
? this.state.pipelineJson.settings.auto_eviction ==
true
: false
}
})()}
onChange={this.onChangeEviction.bind(this)}
label="Automatic memory eviction"
/>
</div>
);
}
})()}
</div>

<MDCButtonReact
label="save"
classNames={["mdc-button--raised"]}
onClick={this.saveGeneralForm.bind(this)}
/>
</form>

<h3 className="push-up push-down">Data passing</h3>

<MDCButtonReact
disabled={this.state.restartingMemoryServer}
label="Clear memory"
icon="memory"
classNames={["mdc-button--raised"]}
onClick={this.restartMemoryServer.bind(this)}
/>

{(() => {
if (this.state.restartingMemoryServer) {
return <p className="push-up">Restarting in progress...</p>;
}
})()}
</div>
);
} else {
return <MDCLinearProgressReact />;
}
})()}

<MDCButtonReact
classNames={["close-button"]}
Expand Down

0 comments on commit 95bc2aa

Please sign in to comment.