Skip to content

Commit

Permalink
Merge pull request #911 from sandialabs/dac-couch-changes
Browse files Browse the repository at this point in the history 
Dac couch changes
  • Loading branch information
@Mletter1
Mletter1 committed Sep 18, 2019
2 parents 5451e91 + 1667767 commit 53012c3
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 82 deletions.
53 changes: 34 additions & 19 deletions packages/slycat/web/server/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import threading
import base64
import six
import time

config = {}
cache_it = Cache(seconds=1000000) # 277.777778 hours
Expand Down Expand Up @@ -297,17 +298,19 @@ def put_model_arrayset(database, model, aid, input=False):
:return:
"""
with get_model_lock(model["_id"]):
model = database.get('model',model["_id"])
slycat.web.server.update_model(database, model, message="Starting array set %s." % (aid))
storage = uuid.uuid4().hex
with slycat.web.server.hdf5.lock:
with slycat.web.server.hdf5.create(storage) as file:
arrayset = slycat.hdf5.start_arrayset(file)
database.save({"_id": storage, "type": "hdf5"})
model["artifact:%s" % aid] = storage
model["artifact-types"][aid] = "hdf5"
if input:
model["input-artifacts"] = list(set(model["input-artifacts"] + [aid]))
database.save(model)
storage = uuid.uuid4().hex
with slycat.web.server.hdf5.lock:
with slycat.web.server.hdf5.create(storage) as file:
arrayset = slycat.hdf5.start_arrayset(file)
database.save({"_id": storage, "type": "hdf5"})
model = database.get('model',model["_id"])
model["artifact:%s" % aid] = storage
model["artifact-types"][aid] = "hdf5"
if input:
model["input-artifacts"] = list(set(model["input-artifacts"] + [aid]))
database.save(model)


def put_model_array(database, model, aid, array_index, attributes, dimensions):
Expand All @@ -323,11 +326,12 @@ def put_model_array(database, model, aid, array_index, attributes, dimensions):
:return:
"""
with get_model_lock(model["_id"]):
model = database.get('model', model['_id'])
slycat.web.server.update_model(database, model, message="Starting array set %s array %s." % (aid, array_index))
storage = model["artifact:%s" % aid]
with slycat.web.server.hdf5.lock:
with slycat.web.server.hdf5.open(storage, "r+") as file:
slycat.hdf5.ArraySet(file).start_array(array_index, dimensions, attributes)
storage = model["artifact:%s" % aid]
with slycat.web.server.hdf5.lock:
with slycat.web.server.hdf5.open(storage, "r+") as file:
slycat.hdf5.ArraySet(file).start_array(array_index, dimensions, attributes)


def put_model_arrayset_data(database, model, aid, hyperchunks, data):
Expand Down Expand Up @@ -356,6 +360,7 @@ def put_model_arrayset_data(database, model, aid, hyperchunks, data):
data = iter(data)

with get_model_lock(model["_id"]):
model = database.get('model', model['_id'])
slycat.web.server.update_model(database, model, message="Storing data to array set %s." % (aid))

with slycat.web.server.hdf5.lock:
Expand Down Expand Up @@ -463,11 +468,21 @@ def put_project_data_parameter(database, project_data, aid, value, input=False):

def put_model_parameter(database, model, aid, value, input=False):
with get_model_lock(model["_id"]):
model["artifact:%s" % aid] = value
model["artifact-types"][aid] = "json"
if input:
model["input-artifacts"] = list(set(model["input-artifacts"] + [aid]))
database.save(model)
try:
model = database.get('model',model['_id'])
model["artifact:%s" % aid] = value
model["artifact-types"][aid] = "json"
if input:
model["input-artifacts"] = list(set(model["input-artifacts"] + [aid]))
database.save(model)
except Exception as e:
time.sleep(1)
model = database.get('model',model['_id'])
model["artifact:%s" % aid] = value
model["artifact-types"][aid] = "json"
if input:
model["input-artifacts"] = list(set(model["input-artifacts"] + [aid]))
database.save(model)


def delete_model_parameter(database, model, aid):
Expand Down
131 changes: 68 additions & 63 deletions packages/slycat/web/server/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,38 +483,42 @@ def create_project_data(mid, aid, file):
content_type = "text/csv"
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", mid)
project = database.get("project", model["project"])
pid = project["_id"]
timestamp = time.time()
formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
did = uuid.uuid4().hex
#TODO review how we pass files to this
# our file got passed as a list by someone...
if isinstance(file, list):
file = file[0]
#TODO review how we pass aids to this
# if true we are using the new file naming structure
if len(aid) > 1:
# looks like we got passed a list(list()) lets extract the name
if isinstance(aid[1], list):
aid[1] = aid[1][0]
# for backwards compatibility for not passing the file name
else:
aid.append("unnamed_file")
data = {
"_id": did,
"type": "project_data",
"file_name": formatted_timestamp + "_" + aid[1],
"data_table": aid[0],
"project": pid,
"mid": [mid],
"created": datetime.datetime.utcnow().isoformat(),
"creator": cherrypy.request.login,
}
if "project_data" not in model:
model["project_data"] = []
model["project_data"].append(did)
database.save(model)

with slycat.web.server.get_model_lock(model["_id"]):
project = database.get("project", model["project"])
pid = project["_id"]
timestamp = time.time()
formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
did = uuid.uuid4().hex

#TODO review how we pass files to this
# our file got passed as a list by someone...
if isinstance(file, list):
file = file[0]
#TODO review how we pass aids to this
# if true we are using the new file naming structure
if len(aid) > 1:
# looks like we got passed a list(list()) lets extract the name
if isinstance(aid[1], list):
aid[1] = aid[1][0]
# for backwards compatibility for not passing the file name
else:
aid.append("unnamed_file")
data = {
"_id": did,
"type": "project_data",
"file_name": formatted_timestamp + "_" + aid[1],
"data_table": aid[0],
"project": pid,
"mid": [mid],
"created": datetime.datetime.utcnow().isoformat(),
"creator": cherrypy.request.login,
}
if "project_data" not in model:
model["project_data"] = []
model["project_data"].append(did)
model = database.get('model', model["_id"])
database.save(model)
database.save(data)
database.put_attachment(data, filename="content", content_type=content_type, content=file)
cherrypy.log.error("[MICROSERVICE] Added project data %s." % data["file_name"])
Expand Down Expand Up @@ -696,43 +700,45 @@ def model_sensitive_command(mid, type, command):
def put_model(mid):
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", mid)
project = database.get("project", model["project"])
slycat.web.server.authentication.require_project_writer(project)

save_model = False
for key, value in cherrypy.request.json.items():
if key not in ["name", "description", "state", "result", "progress", "message", "started", "finished",
"marking", "bookmark"]:
cherrypy.log.error("slycat.web.server.handlers.py put_model",
"cherrypy.HTTPError 400 unknown model parameter: %s" % key)
raise cherrypy.HTTPError("400 Unknown model parameter: %s" % key)
with slycat.web.server.get_model_lock(model["_id"]):
project = database.get("project", model["project"])
slycat.web.server.authentication.require_project_writer(project)

if key in ["started", "finished"]:
try:
datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f")
except:
save_model = False
model = database.get('model', model["_id"])
for key, value in cherrypy.request.json.items():
if key not in ["name", "description", "state", "result", "progress", "message", "started", "finished",
"marking", "bookmark"]:
cherrypy.log.error("slycat.web.server.handlers.py put_model",
"cherrypy.HTTPError 400 timestamp fields must use ISO-8601.")
raise cherrypy.HTTPError("400 Timestamp fields must use ISO-8601.")
"cherrypy.HTTPError 400 unknown model parameter: %s" % key)
raise cherrypy.HTTPError("400 Unknown model parameter: %s" % key)

if value != model.get(key):
model[key] = value
save_model = True
if key in ["started", "finished"]:
try:
datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f")
except:
cherrypy.log.error("slycat.web.server.handlers.py put_model",
"cherrypy.HTTPError 400 timestamp fields must use ISO-8601.")
raise cherrypy.HTTPError("400 Timestamp fields must use ISO-8601.")

# Revisit this, how booksmark IDs get added to model
if value != model.get(key):
model[key] = value
save_model = True

#if key == "bookmark":
# model[key].append(value)
# save_model = True
#else:
# model[key] = value
# save_model = True
# Revisit this, how booksmark IDs get added to model

model[key] = value
save_model = True
#if key == "bookmark":
# model[key].append(value)
# save_model = True
#else:
# model[key] = value
# save_model = True

if save_model:
database.save(model)
# model[key] = value
# save_model = True

if save_model:
database.save(model)


def post_model_finish(mid):
Expand Down Expand Up @@ -1077,7 +1083,6 @@ def put_project_data_parameter(did, aid):
slycat.web.server.put_project_data_parameter(database, project_data, aid, value, input)



@cherrypy.tools.json_in(on=True)
def put_model_parameter(mid, aid):
database = slycat.web.server.database.couchdb.connect()
Expand Down

0 comments on commit 53012c3

Please sign in to comment.