Merge pull request #18 from ohsu-comp-bio/schema-0.3

TES v0.3.0

README.md

@@ -24,8 +24,8 @@ import tes
 task = tes.Task(
     executors=[
         Executor(
-            image_name="alpine",
-            cmd=["echo", "hello"]
+            image="alpine",
+            command=["echo", "hello"]
         )
     ]
 )

tes/__init__.py

@@ -3,9 +3,9 @@
 from tes.client import HTTPClient
 from tes.utils import unmarshal
 from tes.models import (
-    TaskParameter,
+    Input,
+    Output,
     Resources,
-    Ports,
     Executor,
     Task,
     ExecutorLog,
@@ -22,9 +22,9 @@
 __all__ = [
     HTTPClient,
     unmarshal,
-    TaskParameter,
+    Input,
+    Output,
     Resources,
-    Ports,
     Executor,
     Task,
     ExecutorLog,
@@ -38,4 +38,4 @@
     ServiceInfo
 ]
 
-__version__ = "0.1.6"
+__version__ = "0.2.0"

tes/models.py

@@ -2,6 +2,7 @@
 
 import dateutil.parser
 import json
+import os
 import six
 
 from attr import asdict, attrs, attrib
@@ -101,7 +102,7 @@ def as_json(self, drop_empty=True):
 
 
 @attrs
-class TaskParameter(Base):
+class Input(Base):
     url = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
@@ -117,7 +118,26 @@ class TaskParameter(Base):
     description = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
-    contents = attrib(
+    content = attrib(
+        default=None, convert=strconv, validator=optional(instance_of(str))
+    )
+
+
+@attrs
+class Output(Base):
+    url = attrib(
+        default=None, convert=strconv, validator=optional(instance_of(str))
+    )
+    path = attrib(
+        default=None, convert=strconv, validator=optional(instance_of(str))
+    )
+    type = attrib(
+        default="FILE", validator=in_(["FILE", "DIRECTORY"])
+    )
+    name = attrib(
+        default=None, convert=strconv, validator=optional(instance_of(str))
+    )
+    description = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
 
@@ -130,7 +150,7 @@ class Resources(Base):
     ram_gb = attrib(
         default=None, validator=optional(instance_of((float, int)))
     )
-    size_gb = attrib(
+    disk_gb = attrib(
         default=None, validator=optional(instance_of((float, int)))
     )
     preemptible = attrib(
@@ -141,18 +161,12 @@ class Resources(Base):
     )
 
 
-@attrs
-class Ports(Base):
-    container = attrib(validator=instance_of(int))
-    host = attrib(default=0, validator=instance_of(int))
-
-
 @attrs
 class Executor(Base):
-    image_name = attrib(
+    image = attrib(
         convert=strconv, validator=instance_of(str)
     )
-    cmd = attrib(
+    command = attrib(
         convert=strconv, validator=list_of(str)
     )
     workdir = attrib(
@@ -167,10 +181,7 @@ class Executor(Base):
     stderr = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
-    ports = attrib(
-        default=None, validator=optional(list_of(Ports))
-    )
-    environ = attrib(
+    env = attrib(
         default=None, validator=optional(instance_of(dict))
     )
 
@@ -196,12 +207,6 @@ class ExecutorLog(Base):
     exit_code = attrib(
         default=None, validator=optional(instance_of(int))
     )
-    host_ip = attrib(
-        default=None, convert=strconv, validator=optional(instance_of(str))
-    )
-    ports = attrib(
-        default=None, validator=optional(list_of(Ports))
-    )
 
 
 @attrs
@@ -238,6 +243,9 @@ class TaskLog(Base):
     outputs = attrib(
         default=None, validator=optional(list_of(OutputFileLog))
     )
+    system_logs = attrib(
+        default=None, validator=optional(list_of(str))
+    )
 
 
 @attrs
@@ -249,23 +257,20 @@ class Task(Base):
         default=None,
         validator=optional(in_(
             ["UKNOWN", "QUEUED", "INITIALIZING", "RUNNING", "COMPLETE",
-             "PAUSED", "CANCELED", "ERROR", "SYSTEM_ERROR"]
+             "CANCELED", "EXECUTOR_ERROR", "SYSTEM_ERROR"]
         ))
     )
     name = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
-    project = attrib(
-        default=None, convert=strconv, validator=optional(instance_of(str))
-    )
     description = attrib(
         default=None, convert=strconv, validator=optional(instance_of(str))
     )
     inputs = attrib(
-        default=None, validator=optional(list_of(TaskParameter))
+        default=None, validator=optional(list_of(Input))
     )
     outputs = attrib(
-        default=None, validator=optional(list_of(TaskParameter))
+        default=None, validator=optional(list_of(Output))
     )
     resources = attrib(
         default=None, validator=optional(instance_of(Resources))
@@ -282,44 +287,74 @@ class Task(Base):
     logs = attrib(
         default=None, validator=optional(list_of(TaskLog))
     )
+    creation_time = attrib(
+        default=None,
+        convert=timestampconv,
+        validator=optional(instance_of(datetime))
+    )
 
     def is_valid(self):
-        if self.executors is None:
-            return False, TypeError("executors NoneType")
+        errs = []
+        if self.executors is None or len(self.executors) == 0:
+            errs.append("Must provide one or more Executors")
         else:
             for e in self.executors:
-                if e.environ is not None:
-                    for k, v in self.executors.environ:
+                if e.image is None:
+                    errs.append("Executor image must be provided")
+                if len(e.command) == 0:
+                    errs.append("Executor command must be provided")
+                if e.stdin is not None:
+                    if not os.path.isabs(e.stdin):
+                        errs.append("Executor stdin must be an absolute path")
+                if e.stdout is not None:
+                    if not os.path.isabs(e.stdout):
+                        errs.append("Executor stdout must be an absolute path")
+                if e.stderr is not None:
+                    if not os.path.isabs(e.stderr):
+                        errs.append("Executor stderr must be an absolute path")
+                if e.env is not None:
+                    for k, v in self.executors.env:
                         if not isinstance(k, str) and not isinstance(k, str):
-                            return False, TypeError(
-                                "keys and values of environ must be StrType"
+                            errs.append(
+                                "Executor env keys and values must be StrType"
                             )
 
         if self.inputs is not None:
             for i in self.inputs:
-                if i.url is None and i.contents is None:
-                    return False, TypeError(
-                        "TaskParameter url must be provided"
-                    )
-                if i.url is not None and i.contents is not None:
-                    return False, TypeError(
-                        "TaskParameter url and contents are mutually exclusive"
-                    )
-                if i.url is None and i.path is None:
-                    return False, TypeError(
-                        "TaskParameter url and path must be provided"
-                    )
+                if i.url is None and i.content is None:
+                    errs.append("Input url must be provided")
+                if i.url is not None and i.content is not None:
+                    errs.append("Input url and content are mutually exclusive")
+                if i.path is None:
+                    errs.append("Input path must be provided")
+                elif not os.path.isabs(i.path):
+                    errs.append("Input path must be absolute")
 
         if self.outputs is not None:
             for o in self.outputs:
-                if o.url is None or o.path is None:
-                    return False, TypeError(
-                        "TaskParameter url and path must be provided"
-                    )
-                if o.contents is not None:
-                    return False, TypeError(
-                        "Output TaskParameter instances do not have contents"
+                if o.url is None:
+                    errs.append("Output url must be provided")
+                if o.path is None:
+                    errs.append("Output path must be provided")
+                elif not os.path.isabs(i.path):
+                    errs.append("Output path must be absolute")
+
+        if self.volumes is not None:
+            if len(self.volumes) > 0:
+                for v in self.volumes:
+                    if not os.path.isabs(v):
+                        errs.append("Volume paths must be absolute")
+
+        if self.tags is not None:
+            for k, v in self.tags:
+                if not isinstance(k, str) and not isinstance(k, str):
+                    errs.append(
+                        "Tag keys and values must be StrType"
                     )
+
+        if len(errs) > 0:
+            return False, TypeError("\n".join(errs))
+
         return True, None
 
 

tes/utils.py

@@ -4,7 +4,7 @@
 import re
 
 from requests import HTTPError
-from tes.models import (Task, TaskParameter, Resources, Executor, Ports,
+from tes.models import (Task, Input, Output, Resources, Executor,
                         TaskLog, ExecutorLog, OutputFileLog)
 
 
@@ -44,10 +44,9 @@ def unmarshal(j, o, convert_camel_case=True):
 
     omap = {
         "tasks": Task,
-        "inputs": TaskParameter,
-        "outputs": (TaskParameter, OutputFileLog),
+        "inputs": Input,
+        "outputs": (Output, OutputFileLog),
         "logs": (TaskLog, ExecutorLog),
-        "ports": Ports,
         "resources": Resources,
         "executors": Executor
     }

tests/test_client.py

@@ -12,8 +12,8 @@ class TestHTTPClient(unittest.TestCase):
     task = Task(
         executors=[
             Executor(
-                image_name="alpine",
-                cmd=["echo", "hello"]
+                image="alpine",
+                command=["echo", "hello"]
             )
         ]
     )

tests/test_models.py

@@ -1,24 +1,24 @@
 import json
 import unittest
 
-from tes.models import Task, Executor, TaskParameter, strconv
+from tes.models import Task, Executor, Input, Output, strconv
 
 
 class TestModels(unittest.TestCase):
     task = Task(
         executors=[
             Executor(
-                image_name="alpine",
-                cmd=["echo", "hello"]
+                image="alpine",
+                command=["echo", "hello"]
             )
         ]
     )
 
     expected = {
         "executors": [
             {
-                "image_name": "alpine",
-                "cmd": ["echo", "hello"]
+                "image": "alpine",
+                "command": ["echo", "hello"]
             }
         ]
     }
@@ -30,16 +30,16 @@ def test_strconv(self):
         self.assertTrue(strconv(1), 1)
 
         with self.assertRaises(TypeError):
-            TaskParameter(
-                url="s3:/some/path", path="foo", contents=123
+            Input(
+                url="s3:/some/path", path="/opt/foo", content=123
             )
 
     def test_list_of(self):
         with self.assertRaises(TypeError):
             Task(
                 inputs=[
-                    TaskParameter(
-                        url="s3:/some/path", path="foo", contents="content"
+                    Input(
+                        url="s3:/some/path", path="/opt/foo"
                     ),
                     "foo"
                 ]
@@ -55,13 +55,13 @@ def test_is_valid(self):
         self.assertTrue(self.task.is_valid()[0])
 
         task2 = self.task
-        task2.inputs = [TaskParameter(path="foo")]
+        task2.inputs = [Input(path="/opt/foo")]
         self.assertFalse(task2.is_valid()[0])
 
         task3 = self.task
         task3.outputs = [
-            TaskParameter(
-                url="s3:/some/path", path="foo", contents="content"
+            Output(
+                url="s3:/some/path", path="foo"
             )
         ]
         self.assertFalse(task3.is_valid()[0])