Merge pull request #38 from aipescience/uws11_update

Uws1.1 update

uws/UWS/client.py

@@ -66,7 +66,7 @@ def _validate_and_parse_filters(self, filters):
 
             try:
                 date = dateutil.parser.parse(after)
-                # The day defaults to current day, not to '01', if no day is 
+                # The day defaults to current day, not to '01', if no day is
                 # given (e.g. '2010-09'->'2010-09-06').
                 # Let's tell the user how the given value was interpreted:
                 if str(date) != str(after):
@@ -79,7 +79,7 @@ def _validate_and_parse_filters(self, filters):
             if date.utcoffset() is not None:
                 utz = pytz.timezone('UTC')
                 date = date.astimezone(utz).replace(tzinfo=None)
-                print "Note: Date time was converted to UTC time: %s" %(str(date))
+                print "Note: Date time was converted to UTC time: %s" % (str(date))
 
             date = date.isoformat()
             params.append(("AFTER", date))
@@ -88,13 +88,12 @@ def _validate_and_parse_filters(self, filters):
             try:
                 last = int(last)
             except:
-                raise UWSError("Value for 'last' argument must be a (positive) integer: %s" % (str(last)))
+                raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last)))
 
-            if last < 0:
-                raise UWSError("Value for 'last' argument must be a (positive) integer: %s" % (str(last)))
+            if last < 1:
+                raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last)))
             params.append(("LAST", last))
 
-
         return params
 
     def _validate_and_parse_wait(self, wait, phase=None):
@@ -121,7 +120,7 @@ def get_job(self, id, wait=None, phase=None):
         try:
             response = self.connection.get(id, params)
         except:
-            # Do not make a second request wihtout params, throw error
+            # Do not make a second request without params, throw error
             # immediately
             raise UWSError(str(e))
 
@@ -143,7 +142,7 @@ def get_phase(self, id):
 
         raw = response.read()
         result = raw
-        
+
         return result
 
     def new_job(self, args={}):

uws/UWS/connection.py

@@ -78,7 +78,7 @@ def get(self, path, params=None):
             raise RuntimeError('Resource does not exist')
 
         if response.status != 200:
-            raise RuntimeError('Error while connection to server: Got response: %s %s' % (response.status, response.reason))
+            raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason))
 
         return response
 
@@ -126,7 +126,7 @@ def post(self, path, args):
             raise RuntimeError('Resource does not exist')
 
         if response.status != 200:
-            raise RuntimeError('Error while connection to server: Got response: %s %s' % (response.status, response.reason))
+            raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason))
 
         return response
 
@@ -167,7 +167,7 @@ def delete(self, path):
             raise RuntimeError('Resource does not exist')
 
         if response.status != 200:
-            raise RuntimeError('Error while connection to server: Got response: %s %s' % (response.status, response.reason))
+            raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason))
 
         return response
 

uws/UWS/models.py

@@ -5,13 +5,14 @@
 #uws_2_namespace = "http://www.ivoa.net/xml/UWS/v2.0"
 xlink_namespace = "http://www.w3.org/1999/xlink"
 
-class UWS1Flavour(object): 
+
+class UWS1Flavour(object):
     def __init__(self, namespaces=None):
 
         if uws_1_namespace not in namespaces.values():
             raise RuntimeError("No supported UWS namespace found in xml-response, cannot parse xml.")
 
-        # prepend each element's name with the correct uws-namespace 
+        # prepend each element's name with the correct uws-namespace
         # for this version
         self.uws_namespace = uws_1_namespace
         self.jobs = et.QName(self.uws_namespace, "jobs")
@@ -21,6 +22,7 @@ def __init__(self, namespaces=None):
         self.runId = et.QName(self.uws_namespace, "runId")
         self.ownerId = et.QName(self.uws_namespace, "ownerId")
         self.quote = et.QName(self.uws_namespace, "quote")
+        self.creationTime = et.QName(self.uws_namespace, "creationTime")
         self.startTime = et.QName(self.uws_namespace, "startTime")
         self.endTime = et.QName(self.uws_namespace, "endTime")
         self.executionDuration = et.QName(self.uws_namespace, "executionDuration")
@@ -129,12 +131,18 @@ def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_names
         self.reference = Reference()
         self.phase = []
 
-        if xml_node is not None:
+        if xml_node is not None:  # When should this ever be None?????
             self.id = xml_node.get('id')
 
             # UWS standard defines array, therefore treat phase as array
+            # (... actually it does not, but keep it anyway like this, maybe at
+            # some point in the future all phases of a job are provided as list)
             self.phase = [elm.text for elm in xml_node.findall(uws_flavour.phase)]
             self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace)
+            self.runId = xml_node.get('runId')
+            self.ownerId = xml_node.get('ownerId')
+            self.creationTime = xml_node.get('creationTime')
+
         elif id is not None and phase is not None and reference is not None:
             self.id = id
 
@@ -194,6 +202,7 @@ def __init__(self, xml=None):
         self.owner_id = None
         self.phase = ["PENDING"]
         self.quote = None
+        self.creation_time = None
         self.start_time = None
         self.end_time = None
         self.execution_duration = 0
@@ -216,10 +225,10 @@ def __init__(self, xml=None):
             self.job_id = parsed.find(uws_flavour.jobId).text
 
             self.run_id = self._get_optional(parsed, uws_flavour.runId)
-
             self.owner_id = parsed.find(uws_flavour.ownerId).text
             self.phase = [parsed.find(uws_flavour.phase).text]
             self.quote = self._get_optional(parsed, uws_flavour.quote)
+            self.creation_time = self._get_optional(parsed, uws_flavour.creationTime)
             self.start_time = parsed.find(uws_flavour.startTime).text
             self.end_time = parsed.find(uws_flavour.endTime).text
             self.execution_duration = int(parsed.find(uws_flavour.executionDuration).text)
@@ -255,6 +264,7 @@ def __unicode__(self):
         str += "OwnerId : '%s'\n" % self.owner_id
         str += "Phase : '%s'\n" % ', '.join(self.phase)
         str += "Quote : '%s'\n" % self.quote
+        str += "CreationTime : '%s'\n" % self.creation_time
         str += "StartTime : '%s'\n" % self.start_time
         str += "EndTime : '%s'\n" % self.end_time
         str += "ExecutionDuration : '%s'\n" % self.execution_duration

uws/UWS/tests/test_models.py

@@ -110,7 +110,7 @@ def setUp(self):
     def test(self):
         job = UWS.models.Job(self.xml)
 
-        job_str = "JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'COMPLETED'\nQuote : 'None'\nStartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 )  GROUP BY FLOOR( LOG10( Mvir ) / 0.25 )  ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,  `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262`  GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nResult id 'csv' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\njobInfo :\n"
+        job_str = "JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'COMPLETED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 )  GROUP BY FLOOR( LOG10( Mvir ) / 0.25 )  ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,  `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262`  GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nResult id 'csv' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\njobInfo :\n"
         self.assertEqual(str(job), job_str)
 
         self.assertEqual(job.job_id, '335912448787925')
@@ -209,7 +209,7 @@ def setUp(self):
     def test(self):
         job = UWS.models.Job(self.xml)
 
-        job_str = "JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ABORTED'\nQuote : 'None'\nStartTime : '2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 )   ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,  SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551`   \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n"
+        job_str = "JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ABORTED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 )   ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,  SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551`   \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n"
         self.assertEqual(str(job), job_str)
 
         self.assertEqual(job.job_id, '308893189727250')
@@ -291,7 +291,7 @@ def setUp(self):
     def test(self):
         job = UWS.models.Job(self.xml)
 
-        job_str = "JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ERROR'\nQuote : 'None'\nStartTime : '2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT  COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,   (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512`   \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n Error Summary - type 'transient' hasDetail: False - message: Remote MySQL server has gone away\njobInfo :\n"
+        job_str = "JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ERROR'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT  COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`,   (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512`   \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n Error Summary - type 'transient' hasDetail: False - message: Remote MySQL server has gone away\njobInfo :\n"
         self.assertEqual(str(job), job_str)
 
         self.assertEqual(job.job_id, '1177277256137938')
@@ -346,7 +346,7 @@ def setUp(self):
         self.xml = '''
 <?xml version="1.0" encoding="UTF-8"?>
 <jobs xmlns="http://www.ivoa.net/xml/UWS/v1.0" xmlns:xlink="http://www.w3.org/1999/xlink">
-  <jobref id="2014-06-03T15:33:29:4235" xlink:href="https://www.cosmosim.org/uws/query/335912448787925" xlink:type="simple">
+  <jobref id="335912448787925" xlink:href="https://www.cosmosim.org/uws/query/335912448787925" xlink:type="simple">
     <phase>COMPLETED</phase>
   </jobref>
 </jobs>
@@ -357,21 +357,22 @@ def test(self):
 
         self.assertEqual(len(job_list.job_reference), 1)
 
-        job_list_str = "Job '2014-06-03T15:33:29:4235' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\n"
+        job_list_str = "Job '335912448787925' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\n"
         self.assertEqual(str(job_list), job_list_str)
 
         job1 = job_list.job_reference[0]
-        self.assertEqual(job1.id, '2014-06-03T15:33:29:4235')
+        self.assertEqual(job1.id, '335912448787925')
         self.assertEqual(job1.phase, ['COMPLETED'])
         self.assertEqual(job1.reference.type, "simple")
         self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925")
 
+
 class JobListNamespace2Test(JobListNamespaceTest):
     def setUp(self):
         self.xml = '''
 <?xml version="1.0" encoding="UTF-8"?>
 <u:jobs xmlns:u="http://www.ivoa.net/xml/UWS/v1.0" xmlns:xlink="http://www.w3.org/1999/xlink">
-  <u:jobref id="2014-06-03T15:33:29:4235" xlink:href="https://www.cosmosim.org/uws/query/335912448787925" xlink:type="simple">
+  <u:jobref id="335912448787925" xlink:href="https://www.cosmosim.org/uws/query/335912448787925" xlink:type="simple">
     <u:phase>COMPLETED</u:phase>
   </u:jobref>
 </u:jobs>