Objectmodel extensions

Enable GridInfo and ProcessingProgram model object for developers

docs/api.rst

@@ -30,3 +30,10 @@ ProcessingJob
 .. automodule:: ispyb.model.processingjob
     :members:
 
+ProcessingProgram
+=================
+
+These objects correspond to entries in the ISPyB table AutoProcProgram.
+
+.. automodule:: ispyb.model.processingprogram
+    :members:

ispyb/__init__.py

@@ -6,7 +6,7 @@
   import ConfigParser as configparser
 import logging
 
-__version__ = '4.10.0'
+__version__ = '4.11.0'
 
 _log = logging.getLogger('ispyb')
 

ispyb/model/__future__.py

@@ -0,0 +1,101 @@
+from __future__ import absolute_import, division, print_function
+
+# Enables direct database functions in places where stored procedures are not
+# yet available. To use, run:
+#
+# import ispyb.model.__future__
+# ispyb.model.__future__.enable('/path/to/.../database.cfg')
+
+try:
+  import configparser
+except ImportError:
+  import ConfigParser as configparser
+import mysql.connector
+
+def enable(configuration_file):
+  global _db, _db_cc
+  '''Enable access to features that are currently under development.'''
+
+  cfgparser = configparser.RawConfigParser()
+  if not cfgparser.read(configuration_file):
+    raise RuntimeError('Could not read from configuration file %s' % configuration_file)
+  cfgsection = dict(cfgparser.items('ispyb'))
+  host = cfgsection.get('host')
+  port = cfgsection.get('port', 3306)
+  database = cfgsection.get('database')
+  username = cfgsection.get('username')
+  password = cfgsection.get('password')
+
+  # Open a direct MySQL connection
+  _db = mysql.connector.connect(host=host, port=port, user=username, password=password, database=database)
+  _db_cc = DictionaryContextcursorFactory(_db.cursor)
+
+  import ispyb.model.gridinfo
+  ispyb.model.gridinfo.GridInfo.reload = _get_gridinfo
+  import ispyb.model.processingprogram
+  ispyb.model.processingprogram.ProcessingProgram.reload = _get_autoprocprogram
+
+class DictionaryContextcursorFactory(object):
+  '''This class creates dictionary context manager objects for mysql.connector
+     cursors. By using a context manager it is ensured that cursors are
+     closed immediately after use.
+     Context managers created via this factory return results as a dictionary
+     by default, and offer a .run() function, which is an alias to .execute
+     that accepts query parameters as function parameters rather than a list.
+  '''
+
+  def __init__(self, cursor_factory_function):
+    '''Set up the context manager factory.'''
+
+    class ContextManager(object):
+      '''The context manager object which is actually used in the
+            with .. as ..:
+         clause.'''
+
+      def __init__(cm, parameters):
+        '''Store any constructor parameters, given as dictionary, so that they
+           can be passed to the cursor factory later.'''
+        cm.cursorparams = { 'dictionary': True }
+        cm.cursorparams.update(parameters)
+
+      def __enter__(cm):
+        '''Enter context. Instantiate and return the actual cursor using the
+           given constructor, parameters, and an extra .run() function.'''
+        cm.cursor = cursor_factory_function(**cm.cursorparams)
+
+        def flat_execute(stmt, *parameters):
+          '''Pass all given function parameters as a list to the existing
+             .execute() function.'''
+          return cm.cursor.execute(stmt, parameters)
+        setattr(cm.cursor, 'run', flat_execute)
+        return cm.cursor
+
+      def __exit__(cm, *args):
+        '''Leave context. Close cursor. Destroy reference.'''
+        cm.cursor.close()
+        cm.cursor = None
+
+    self._contextmanager_factory = ContextManager
+
+  def __call__(self, **parameters):
+    '''Creates and returns a context manager object.'''
+    return self._contextmanager_factory(parameters)
+
+def _get_gridinfo(self):
+  with _db_cc() as cursor:
+    cursor.run("SELECT * "
+               "FROM GridInfo "
+               "WHERE dataCollectionGroupId = %s "
+               "LIMIT 1;", self._dcgid)
+    self._data = cursor.fetchone()
+
+def _get_autoprocprogram(self):
+  with _db_cc() as cursor:
+    cursor.run("SELECT processingCommandLine as commandLine, processingPrograms as programs, "
+               "processingStatus as status, processingMessage as message, processingEndTime as endTime, "
+               "processingStartTime as startTime, processingEnvironment as environment, "
+               "processingJobId as jobId, recordTimeStamp, autoProcProgramId "
+               "FROM AutoProcProgram "
+               "WHERE autoProcProgramId = %s "
+               "LIMIT 1;", self._appid)
+    self._data = cursor.fetchone()

ispyb/model/datacollection.py

@@ -1,6 +1,10 @@
 from __future__ import absolute_import, division, print_function
 
+import os
+import re
+
 import ispyb.model
+import ispyb.model.gridinfo
 
 class DataCollection(ispyb.model.DBCache):
   '''An object representing a DataCollection database entry. The object
@@ -39,6 +43,28 @@ def group(self):
       self._cache_group = DataCollectionGroup(self.dcgid, self._db.conn)
     return self._cache_group
 
+  @property
+  def file_template_full(self):
+    '''Template for file names with full directory path. As with file_template
+       \'#\' characters stand in for image number digits.'''
+    return os.path.join(self.file_directory, self.file_template)
+
+  @property
+  def file_template_full_python(self):
+    '''Template for file names that can be used in python string templates
+       (for use with the % operator), with %0xd standing in for the x image
+       number digits.'''
+    if not self.file_template_full:
+      return None
+    if '#' not in self.file_template_full:
+      return self.file_template_full
+    return re.sub(
+        r'#+',
+        lambda x: "%%0%dd" % len(x.group(0)),
+        self.file_template_full.replace('%', '%%'),
+        count=1,
+    )
+
   def __repr__(self):
     '''Returns an object representation, including the DataCollectionID,
        the database connection interface object, and the cache status.'''
@@ -58,11 +84,14 @@ def __str__(self):
       '  Started      : {0.time_start}',
       '  Finished     : {0.time_end}',
       '  DC group     : {0.dcgid}',
+      '  Image files  : {0.file_template_full}',
     ))).format(self)
 
 ispyb.model.add_properties(DataCollection, (
     ('dcgid', 'groupId', 'Returns the Data Collection Group ID associated with this data collection. '
         'You can use .group to get the data collection group model object instead'),
+    ('file_template', 'fileTemplate', 'Template for file names with the character \'#\' standing in for image number digits.'),
+    ('file_directory', 'imgDir', 'Fully qualified path to the image files'),
     ('time_start', 'startTime', None),
     ('time_end', 'endTime', None),
     ('image_count', 'noImages', None),
@@ -105,7 +134,7 @@ def dcgid(self):
   def gridinfo(self):
     '''Returns a GridInfo object.'''
     if self._cache_gridinfo is None:
-      self._cache_gridinfo = GridInfo(self.dcgid, self._db)
+      self._cache_gridinfo = ispyb.model.gridinfo.GridInfo(self.dcgid, self._db)
     return self._cache_gridinfo
 
   def __repr__(self):

ispyb/model/gridinfo.py

@@ -28,7 +28,8 @@ def reload(self):
 
   @property
   def dcgid(self):
-    '''Returns the DataCollectionGroupID.'''
+    '''Returns the Data Collection Group ID associated with this grid
+       information.'''
     return self._dcgid
 
   def __repr__(self):
@@ -43,7 +44,24 @@ def __repr__(self):
   def __str__(self):
     '''Returns a pretty-printed object representation.'''
     if not self.cached:
-      return 'GridInfo #%d (not yet loaded from database)' % self._dcid
+      return 'GridInfo #%d (not yet loaded from database)' % self._dcgid
     return ('\n'.join((
       'GridInfo #{0.dcgid}',
     ))).format(self)
+
+ispyb.model.add_properties(GridInfo, (
+    ('dx_mm', 'dx_mm', 'Grid element width in mm'),
+    ('dy_mm', 'dy_mm', 'Grid element height in mm'),
+    ('id', 'gridInfoId', 'A unique ID identifying this grid information record'),
+    ('orientation', 'orientation', 'The orientation of the grid, either "horizontal" or "vertical"'),
+    ('pixels_per_micron_x', 'pixelsPerMicronX', 'Number of pixels per micrometre (horizontal) when displaying the grid in GDA'),
+    ('pixels_per_micron_y', 'pixelsPerMicronY', 'Number of pixels per micrometre (vertical) when displaying the grid in GDA'),
+    ('timestamp', 'recordTimeStamp', 'Time and date of record creation'),
+    ('steps_x', 'steps_x', 'Width of the grid scan in number of grid elements'),
+    ('steps_y', 'steps_y', 'Height of the grid scan in number of grid elements'),
+    ('snaked', 'snaked', 'Whether the fast scan axis is inverted (1) or kept (0) for every slow axis acquisition'),
+    ('snapshot_offset_pixel_x', 'snapshot_offsetXPixel',
+         'Horizontal distance from the top left corner in GDA to the first grid element'),
+    ('snapshot_offset_pixel_y', 'snapshot_offsetYPixel',
+         'Vertical distance from the top left corner in GDA to the first grid element'),
+))

ispyb/model/interface.py

@@ -1,22 +1,23 @@
 from __future__ import absolute_import, division, print_function
 
 import ispyb.model.datacollection
+import ispyb.model.processingprogram
 import ispyb.model.processingjob
 
 class ObjectModelMixIn():
   '''Object model accessor functions for Connector classes.'''
 
   def get_data_collection(self, dcid):
     '''Return a DataCollection object representing the information
-       about the selected data collection'''
+       about the selected data collection.'''
     return ispyb.model.datacollection.DataCollection(
         dcid,
         self.mx_acquisition,
     )
 
   def get_data_collection_group(self, dcgid):
     '''Return a DataCollectionGroup object representing the information
-       about the selected data collection group'''
+       about the selected data collection group.'''
     return ispyb.model.datacollection.DataCollectionGroup(
         dcgid,
         self,
@@ -29,3 +30,11 @@ def get_processing_job(self, jobid):
         jobid,
         self.mx_processing,
     )
+
+  def get_processing_program(self, appid):
+    '''Return an ProcessingProgram object representing the information
+       about a processing program invocation.'''
+    return ispyb.model.processingprogram.ProcessingProgram(
+        appid,
+        self,
+    )

ispyb/model/processingjob.py

@@ -3,7 +3,7 @@
 import collections
 
 import ispyb.model
-import ispyb.model.autoprocprogram
+import ispyb.model.processingprogram
 
 class ProcessingJob(ispyb.model.DBCache):
   '''An object representing a ProcessingJob database entry. The object lazily
@@ -20,6 +20,7 @@ def __init__(self, jobid, db_area):
        :return: A ProcessingJob object representing the database entry for the
                 specified job ID
     '''
+    self._cache_dc = None
     self._cache_parameters = None
     self._cache_programs = None
     self._cache_sweeps = None
@@ -32,12 +33,23 @@ def reload(self):
 
   @property
   def DCID(self):
-    '''Returns the data collection id.'''
+    '''Returns the main data collection id.'''
     dcid = self._data['dataCollectionId']
     if dcid is None:
       return None
     return int(dcid)
 
+  @property
+  def data_collection(self):
+    '''Returns the DataCollection model object for the main data collection of
+       the ProcessingJob.'''
+    dcid = self._data['dataCollectionId']
+    if dcid is None:
+      return None
+    if self._cache_dc is None:
+      self._cache_dc = self._db.conn.get_data_collection(self._data['dataCollectionId'])
+    return self._cache_dc
+
   @property
   def jobid(self):
     '''Returns the ProcessingJob ID.'''
@@ -57,6 +69,8 @@ def parameters(self):
 
   @property
   def sweeps(self):
+    '''Returns a list of ProcessingJobImageSweeps involved in this
+       processing job.'''
     if self._cache_sweeps is None:
       self._cache_sweeps = ProcessingJobImageSweeps(self._jobid, self._db)
     return self._cache_sweeps
@@ -183,15 +197,22 @@ class ProcessingJobImageSweep(object):
      sweep id.
   '''
 
-  def __init__(self, dcid, start, end, sweep_id):
+  def __init__(self, dcid, start, end, sweep_id, db_area):
     self._dcid, self._sid = int(dcid), int(sweep_id)
     self._start, self._end = int(start), int(end)
+    self._db = db_area
 
   @property
   def DCID(self):
     '''Returns the data collection id.'''
     return self._dcid
 
+  @property
+  def data_collection(self):
+    '''Returns the DataCollection model object for the data collection of this
+       sweep.'''
+    return self._db.conn.get_data_collection(self._dcid)
+
   @property
   def start(self):
     '''Returns the start image number of the sweep'''
@@ -237,7 +258,7 @@ def reload(self):
     '''Load/update information from the database.'''
     try:
       self._data = [
-          ProcessingJobImageSweep(p['dataCollectionId'], p['startImage'], p['endImage'], p['sweepId'])
+          ProcessingJobImageSweep(p['dataCollectionId'], p['startImage'], p['endImage'], p['sweepId'], self._db)
           for p in self._db.retrieve_job_image_sweeps(self._jobid)
       ]
     except ispyb.exception.ISPyBNoResultException:
@@ -283,7 +304,7 @@ def reload(self):
     '''Load/update information from the database.'''
     try:
       self._data = [
-          ispyb.model.autoprocprogram.AutoProcProgram(p['id'], self._db, preload=p)
+          ispyb.model.processingprogram.ProcessingProgram(p['id'], self._db, preload=p)
           for p in self._db.retrieve_programs_for_job_id(self._jobid)
       ]
     except ispyb.exception.ISPyBNoResultException: