Adding docstrings !

panoramix/models.py

@@ -36,6 +36,10 @@
 
 
 class AuditMixinNullable(AuditMixin):
+
+    """Altering the AuditMixin to use nullable fields, allows creating
+    objects programmatically outside of CRUD"""
+
     created_on = Column(DateTime, default=datetime.now, nullable=True)
     changed_on = Column(
         DateTime, default=datetime.now,
@@ -142,6 +146,7 @@ def json_data(self):
 
     @property
     def slice_url(self):
+        """Defines the url to access the slice"""
         try:
             slice_params = json.loads(self.params)
         except Exception as e:
@@ -175,7 +180,7 @@ def slice_link(self):
 
 class Dashboard(Model, AuditMixinNullable):
 
-    """A dash to slash"""
+    """The dashboard object!"""
 
     __tablename__ = 'dashboards'
     id = Column(Integer, primary_key=True)
@@ -240,6 +245,9 @@ def dttm_cols(self):
 
 
 class Database(Model, AuditMixinNullable):
+
+    """An ORM object that stores Database related information"""
+
     __tablename__ = 'dbs'
     id = Column(Integer, primary_key=True)
     database_name = Column(String(250), unique=True)
@@ -256,15 +264,14 @@ def safe_sqlalchemy_uri(self):
         return self.sqlalchemy_uri
 
     def grains(self):
+        """Defines time granularity database-specific expressions.
 
-        """Defines time granularity database-specific expressions. The idea
-        here is to make it easy for users to change the time grain form a
-        datetime (maybe the source grain is arbitrary timestamps, daily
+        The idea here is to make it easy for users to change the time grain
+        form a datetime (maybe the source grain is arbitrary timestamps, daily
         or 5 minutes increments) to another, "truncated" datetime. Since
         each database has slightly different but similar datetime functions,
         this allows a mapping between database engines and actual functions.
         """
-
         Grain = namedtuple('Grain', 'name function')
         DB_TIME_GRAINS = {
             'presto': (
@@ -314,6 +321,9 @@ def sql_link(self):
 
 
 class SqlaTable(Model, Queryable, AuditMixinNullable):
+
+    """An ORM object for SqlAlchemy table references"""
+
     type = "table"
 
     __tablename__ = 'tables'
@@ -554,6 +564,7 @@ def query(
             df=df, duration=datetime.now() - qry_start_dttm, query=sql)
 
     def fetch_metadata(self):
+        """Fetches the metadata for the table and merges it in"""
         table = self.database.get_table(self.table_name)
         try:
             table = self.database.get_table(self.table_name)
@@ -653,6 +664,9 @@ def fetch_metadata(self):
 
 
 class SqlMetric(Model, AuditMixinNullable):
+
+    """ORM object for metrics, each table can have multiple metrics"""
+
     __tablename__ = 'sql_metrics'
     id = Column(Integer, primary_key=True)
     metric_name = Column(String(512))
@@ -666,6 +680,9 @@ class SqlMetric(Model, AuditMixinNullable):
 
 
 class TableColumn(Model, AuditMixinNullable):
+
+    """ORM object for table columns, each table can have multiple columns"""
+
     __tablename__ = 'table_columns'
     id = Column(Integer, primary_key=True)
     table_id = Column(Integer, ForeignKey('tables.id'))
@@ -693,6 +710,9 @@ def isnum(self):
 
 
 class DruidCluster(Model, AuditMixinNullable):
+
+    """ORM object referencing the Druid clusters"""
+
     __tablename__ = 'clusters'
     id = Column(Integer, primary_key=True)
     cluster_name = Column(String(250), unique=True)
@@ -726,6 +746,9 @@ def refresh_datasources(self):
 
 
 class DruidDatasource(Model, AuditMixinNullable, Queryable):
+
+    """ORM object referencing Druid datasources (tables)"""
+
     type = "druid"
 
     baselink = "datasourcemodelview"
@@ -793,6 +816,7 @@ def get_metric_obj(self, metric_name):
         ][0]
 
     def latest_metadata(self):
+        """Returns segment metadata from the latest segment"""
         client = self.cluster.get_pydruid_client()
         results = client.time_boundary(datasource=self.datasource_name)
         if not results:
@@ -813,6 +837,8 @@ def generate_metrics(self):
 
     @classmethod
     def sync_to_db(cls, name, cluster):
+        """Fetches the metadata for that datasource and merges it into
+        the Panoramix database"""
         print("Syncing Druid datasource [{}]".format(name))
         session = get_session()
         datasource = session.query(cls).filter_by(datasource_name=name).first()
@@ -855,8 +881,11 @@ def query(
             timeseries_limit=None,
             row_limit=None,
             inner_from_dttm=None, inner_to_dttm=None,
-            extras=None,
+            extras=None,  # noqa
             select=None):
+        """Runs a query against Druid and returns a dataframe.
+        This query interface is common to SqlAlchemy and Druid"""
+        # TODO refactor into using a TBD Query object
         qry_start_dttm = datetime.now()
 
         inner_from_dttm = inner_from_dttm or from_dttm
@@ -996,6 +1025,9 @@ def query(
 
 
 class Log(Model):
+
+    """ORM object used to log Panoramix actions to the database"""
+
     __tablename__ = 'logs'
 
     id = Column(Integer, primary_key=True)
@@ -1033,6 +1065,9 @@ def wrapper(*args, **kwargs):
 
 
 class DruidMetric(Model):
+
+    """ORM object referencing Druid metrics for a datasource"""
+
     __tablename__ = 'metrics'
     id = Column(Integer, primary_key=True)
     metric_name = Column(String(512))
@@ -1055,6 +1090,9 @@ def json_obj(self):
 
 
 class DruidColumn(Model):
+
+    """ORM model for storing Druid datasource column metadata"""
+
     __tablename__ = 'columns'
     id = Column(Integer, primary_key=True)
     datasource_name = Column(
@@ -1080,6 +1118,7 @@ def isnum(self):
         return self.type in ('LONG', 'DOUBLE', 'FLOAT')
 
     def generate_metrics(self):
+        """Generate metrics based on the column metadata"""
         M = DruidMetric
         metrics = []
         metrics.append(DruidMetric(

panoramix/utils.py

@@ -1,3 +1,5 @@
+"""Utility functions used across Panoramix"""
+
 from datetime import datetime
 import hashlib
 import functools
@@ -12,10 +14,12 @@
 
 
 class memoized(object):
+
     """Decorator that caches a function's return value each time it is called.
     If called later with the same arguments, the cached value is returned, and
     not re-evaluated.
     """
+
     def __init__(self, func):
         self.func = func
         self.cache = {}
@@ -47,8 +51,7 @@ def list_minus(l, minus):
 
 def parse_human_datetime(s):
     """
-    Use the parsedatetime lib to return ``datetime.datetime`` from human
-    generated strings
+    Returns ``datetime.datetime`` from human readable strings
 
     >>> from datetime import date, timedelta
     >>> from dateutil.relativedelta import relativedelta
@@ -92,8 +95,7 @@ def merge_perm(sm, permission_name, view_menu_name):
 
 def parse_human_timedelta(s):
     """
-    Use the parsedatetime lib to return ``datetime.datetime`` from human
-    generated strings
+    Returns ``datetime.datetime`` from natural language time deltas
 
     >>> parse_human_datetime("now") <= datetime.now()
     True
@@ -107,7 +109,9 @@ def parse_human_timedelta(s):
 
 
 class JSONEncodedDict(TypeDecorator):
+
     """Represents an immutable structure as a json-encoded string."""
+
     impl = TEXT
     def process_bind_param(self, value, dialect):
         if value is not None:
@@ -122,6 +126,9 @@ def process_result_value(self, value, dialect):
 
 
 class ColorFactory(object):
+
+    """Used to generated arrays of colors server side"""
+
     BNB_COLORS = [
         #rausch    hackb      kazan      babu      lima        beach     barol
         '#ff5a5f', '#7b0051', '#007A87', '#00d1c1', '#8ce071', '#ffb400', '#b4a76c',
@@ -134,7 +141,8 @@ def __init__(self, hash_based=False):
         self.hash_based = hash_based
 
     def get(self, s):
-        """
+        """Gets a color from a string and memoize the association
+
         >>> cf = ColorFactory()
         >>> cf.get('item_1')
         '#ff5a5f'
@@ -155,9 +163,7 @@ def get(self, s):
 
 
 def init(panoramix):
-    """
-    Inits the Panoramix application with security roles and such
-    """
+    """Inits the Panoramix application with security roles and such"""
     db = panoramix.db
     models = panoramix.models
     sm = panoramix.appbuilder.sm
@@ -204,9 +210,7 @@ def init(panoramix):
 
 
 def datetime_f(dttm):
-    """
-    Formats datetime to take less room is recent
-    """
+    """Formats datetime to take less room when it is recent"""
     if dttm:
         dttm = dttm.isoformat()
         now_iso = datetime.now().isoformat()