Migrated some base types from nadcon5-ng

nc5ng/types/__init__.py

@@ -0,0 +1,23 @@
+"""
+Common Base Types, Type Generators and Abstract Base Types for ``nc5ng``
+
+DataPoint Types
+---------------
+.. automodule:: nc5ng.types.datapoint
+
+
+Data Parsers
+------------
+
+.. automodule:: nc5ng.types.parsers
+  :members:
+
+
+"""
+
+from .datapoint import DataPointType
+from .parsers import BaseFileParser, FortranFormatFileParser, IndexedFortranFormatFileParser
+
+
+
+__all__ = ['DataPointType']

nc5ng/types/datapoint.py

@@ -0,0 +1,161 @@
+"""Base Datapoint Types
+
+``DataPointType`` Metaclass
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: nc5ng.types.DataPointType
+  :members:
+
+"""
+
+import logging
+
+
+
+class DataPointType(type):
+    """ Metaclass for DataPoints, defines class creation and class/hierarchy member variables 
+    
+    The meta-class in part, hides some of the more rote requirements of our datapoint type
+    from the actual object hierarchy
+
+    This allows some magic like run-time casting to the correct datapoint without knowing the 
+    type specifically, and a persisitent library-wide memory backed database for quick retrieval and to minimize replication
+
+    To understand the meta class, there are a number of tutorials available online, roughly speaking this class is what is used to "create" the DataPoint class, and allows us to manipulate the class without needing any implementation details. 
+
+
+    Defining a new DataPointType Hierarchy simply requires using this class as the metaclass
+
+        class NewDataPoint(metaclass=DataPointType):
+          pass
+
+    By creating a new DataPointType, the following changes will be done to the final class
+
+      - The type will have a database (dictionary) of types registered with the base class
+      - Each type and subtype will have a point container (default set) created
+      - The shorthand name will be generated from the class name
+      - Instance Creation will be overidden and allow creation of any other data type by specifting the type shorthand as an argument
+      
+
+    Class Configuration:
+
+    Each new type has some meta-configuration available
+
+    1. To override data point registration 
+       - Create a `@classmethod`  `__register__(cls, point)` to overide how a new point is registered/saved
+       - Create a class member `_point_store` to change the underlying storage type (from set)
+    2. Override shorthand name by specific '_type_shorthand' explicitly in the class
+    
+    
+    Any class that uses this type as a metaclass will be registered 
+    """
+    @classmethod
+    def __prepare__(metacls, name, bases, **kargs):
+        """ Prepare the new class, here for completeness
+        """
+        logging.debug("Preparing Class %s"%name)
+        return super().__prepare__(name, bases, **kargs)
+
+
+    @property
+    def type_shorthand(cls):
+        """ Get the class shorthand name"""
+        return cls._type_shorthand
+
+    @property
+    def point_store(cls):
+        """ Return the type-specific Point Buffer"""
+        return cls._point_store
+
+    @property
+    def point_database(cls):
+        """ Return the Root Database of all DataPoints in this Hierarchy"""
+        return cls._cbdb
+
+
+    def __new__(metacls, name, bases, namespace,  **kargs):
+        """ Create a new data point type, called on class load
+
+        Creates class attributes level point set for storage
+        
+        metaclass __new__ is executed on load time for every
+        class that uses it, it is executed after __prepare__ which
+        constructs the class object
+        """
+
+
+        logging.debug("Creating Class %s"%name)
+        cls = super().__new__(metacls, name, bases, namespace)
+
+        if not(hasattr(cls, '_cbdb')):
+            logging.debug("Creating Data Point Database")
+            cls._cbdb = dict()
+
+
+        # Shorthand Name
+        cls._type_shorthand =  name.lower()
+        while cls._type_shorthand in cls._cbdb:
+            cls._type_shorthand = cls._type_shorthand + "_" # in case of name conflict, add underscore
+
+        # Point Storage
+        cls._point_store = namespace.get('_point_store', set() )
+
+
+        logging.debug("Registering new Data Point Type %s with shorthand %s"%(name, cls._type_shorthand))
+
+
+        cls._cbdb[cls.type_shorthand]={'type':cls, 'points': cls._point_store }
+        return cls
+
+
+    def __init__(cls, name, bases, namespace):
+        """ Initialize a new FileBacked Class
+
+        This is a slot method for class creation, __init__ is called when class is defined (load time)
+
+        \param cls - reference to new type, similiar to @classmethod
+        \param name - new class name
+        \param bases - base classes
+        \param namespace - new class attributes
+        \param Parser - BaseFileParser underlying this type
+        \param **kwargs - keywords passed to Parser initialization
+        
+        """        
+        logging.debug("Creating Data Point Class %s"%name)
+
+        super().__init__(name, bases, namespace)
+
+
+    def __register__(cls, point):
+        """ Register a point with the class buffer """
+        cls._point_store.add(point)
+
+    def __call__(cls, *args,  **kw):
+        """ Create a new Point in this hierarchy"""
+
+        """
+        if typename is not None:
+            if typename not in cls._cbdb:
+                raise TypeError("Invalid Data Point Type with Shorthand %s"%typename)
+            cls = cls._cbdb[typename]['type']
+        """
+        if args and (args[0] in cls.point_database.keys()):
+            typename=args[0]
+        elif kw and ('type' in kw):
+            typename = kw['type']
+        else:
+            typename = cls.type_shorthand
+
+
+        if typename == cls.type_shorthand:
+            point = super().__call__(*args, **kw)
+            if not getattr(point, 'ephemeral', False):
+                cls.__register__(point)
+        elif typename in cls.point_database.keys():
+            point = cls.point_database[typename]['type'].__call__(*args, **kw)
+        else:
+            return None
+
+        return point
+
+

nc5ng/types/parsers.py

@@ -0,0 +1,196 @@
+"""
+Base Parser Types for `nc5ng` submodules
+
+.. automodule:: .
+
+
+
+
+"""
+import fortranformat as ff
+import logging
+from os.path import basename, exists, join, isdir, isfile, isabs
+from os import listdir
+
+
+
+class BaseFileParser(object):
+    """ Base Class for File Parsers
+    
+    
+    
+    """
+    def __init__(self, parser=None, fdir=None, ffile=None):
+        self.parser = parser
+        self.fdir = fdir
+        self.ffile = ffile
+
+
+
+    @property
+    def parser(self):
+        return self._parser
+    @parser.setter
+    def parser(self, value):
+        self._parser = value
+
+    @property
+    def fdir(self):
+        return getattr(self, '_fdir', None)
+
+    @fdir.setter
+    def fdir(self, value):
+        self._fdir = value
+        if (value is not None) and not(isabs(value)):
+            logging.warning("%s is not an absolute path"%str(value))
+
+    @property
+    def ffile(self):
+        return self._ffile
+
+    @ffile.setter
+    def ffile(self, value):
+        self._ffile = value
+
+
+    def __call__(self, it):
+        if self.parser and hasattr(self.parser, '__call__'):
+            return [self.parser(_line) for _line in it]
+        elif self.parser and hasattr(self.parser, 'read'):
+            return [self.parser.read(_line) for _line in it]
+        else:
+            return None
+
+    def __fromfile__(self, f):
+        return {'meta': {}, 'data':[ _ for _ in self(f) if _ is not None] }
+
+    def fromfile(self, ffile=None, process=None, fdir=None):
+        if ffile is None and self.ffile is not None:
+            ffile = self.ffile
+
+        if fdir is None and self.fdir is not None:
+            fdir = self.fdir
+
+        if (fdir is not None) and (ffile is not None) and  not(isabs(ffile)):
+            ffile = join(fdir, ffile)
+
+        with open(ffile,'r') as f:
+            res = self.__fromfile__(f)
+            res['meta']['source'] = ffile
+            return res
+
+        return None
+
+class FortranFormatFileParser(BaseFileParser):
+    """ FileParser for Fortran Fixed Format Files
+    
+    :param fformat: fortran format string
+    :param ffilter: file pre-filter (exclude/include lines)
+    """
+    def __init__(self, fformat = None, ffilter = None):
+        self._parser = None
+        self.fformat = fformat
+        self.ffilter = ffilter
+
+    @property
+    def fformat(self):
+        return self._fformat
+
+    @fformat.setter
+    def fformat(self, value):
+        self._fformat = value
+        if self._fformat:
+            self._parser = ff.FortranRecordReader(self._fformat)
+
+    @property
+    def ffilter(self):
+        return self._ffilter
+
+    @ffilter.setter
+    def ffilter(self, value):
+        if value is None:
+            self._ffilter = lambda x: x
+        else:
+            self._ffilter = value            
+
+    def __call__(self, it):
+        if self._parser:
+            return [ self._ffilter( self._parser.read(_line) ) for _line in it ]
+
+
+
+class IndexedFortranFormatFileParser(FortranFormatFileParser):
+    """ Extentsion for FortranFileParser that allows classes to 
+    switch between pre-registered formats by index
+
+    :param args: Either list of ``[format1,filter1],[format2,filter2],...`` or serial list ``format1, filter1, format2, filter2, ...``
+    :param kwargs: Keyword argument dictionary  ``index:[format,filter]``, dictionary key used for indexing file parser
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        self._running_index=0
+        self._index=0
+        self.indexed_format = {}
+
+        for entry in args:
+            try:
+                fformat,ffilter = entry
+            except TypeException:
+                fformat = args.pop(0)
+                if args: ffilter = args.pop(0)
+                else: ffilter=None
+            self._register_format(fformat, ffilter)
+        for index, entry in kwargs.items():
+            try:
+                fformat, ffilter = entry
+            except TypeException:
+                fformat = entry
+                ffilter = None
+            self._register_format(fformat, ffilter, index)
+        self.index=0
+
+    def _register_format(self, fformat, ffilter=None, index = None, overwrite=False):
+        if index is None:
+            index = self._running_index
+            self.fformat = fformat
+            self._running_index = self._running_index + 1
+
+        if (index not in self.indexed_format or overwrite):
+            self.indexed_format[index]=(fformat,ffilter)
+            if (index == self.index):
+                self.index = index # reload filters
+            return index
+        else:
+            return None
+
+    @property
+    def index(self):
+        return self._index
+
+    @index.setter
+    def index(self, index=None):
+        if index in self.indexed_format:
+            fformat, ffilter = self.indexed_format[index]
+            self.fformat = fformat
+            self.ffilter = ffilter
+            self._index = index
+        else:
+            pass
+
+    def __getitem__(self,index):
+        if index in self.indexed_format:
+            return FortranFormatFileParser(*self.indexed_format[index])
+        else:
+            return None
+
+    def __contains__(self, value):
+        return index in self.indexed_format
+
+    def __call__(self, it, index=None):        
+        if index is None or index == self.index:
+            return super().__call__(it)
+        elif index in self:
+            return p(it)
+        else:
+            return None