Merge branch 'feature-commis' into develop

.gitignore

@@ -64,6 +64,9 @@ target/
 # OS X Stuff
 .DS_Store
 
+# Configuration
+conf/minke.yaml
+
 # Data and fixtures
 fixtures/corpus
 fixtures/tagged

Makefile

@@ -13,6 +13,7 @@ PYTHON_BIN := $(VIRTUAL_ENV)/bin
 # Clean build files
 clean:
 	find . -name "*.pyc" -print0 | xargs -0 rm -rf
+	find . -name "__pycache__" -print0 | xargs -0 rm -rf
 	-rm -rf htmlcov
 	-rm -rf .coverage
 	-rm -rf build

conf/minke-example.yaml

@@ -0,0 +1,23 @@
+##
+## Example configuration file for Minke.
+## Created: Sat Aug 06 21:06:29 2016 -0400
+##
+## Copy this file into one of the following locations, then edit for your
+## specific environment settings, e.g. for production or development.
+##
+## - /etc/minke.yaml
+## - $HOME/.minke.yaml
+## - $(pwd)/conf/minke.yaml
+##
+
+# Base Settings
+debug:      true
+testing:    false
+
+# Notification Settings
+notify:
+    username:    null   # username to the email provider (e.g. gmail address)
+    password:    null   # password to the email provider (e.g. gmail password)
+    email_host:  null   # host name of the email service (e.g. smtp.gmail.com)
+    email_port:  null   # port of the secure smtp (e.g. 587)
+    fail_silent: true   # if there is an error, ignore or raise a new one.

minke/config.py

@@ -0,0 +1,76 @@
+# minke.config
+# Configuration and settings from a YAML file using Confire.
+#
+# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
+# Created:  Sat Aug 06 21:00:05 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: config.py [] benjamin@bengfort.com $
+
+"""
+Configuration and settings from a YAML file using Confire.
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import os
+
+from confire import Configuration
+
+
+##########################################################################
+## Base Paths
+##########################################################################
+
+PROJECT  = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+
+##########################################################################
+## Nested Configurations
+##########################################################################
+
+class NotifyConfiguration(Configuration):
+    """
+    Email settings so that CloudScope can send email messages
+    """
+
+    username    = None
+    password    = None
+    email_host  = None
+    email_port  = None
+    fail_silent = True
+
+
+##########################################################################
+## Minke Configuration
+##########################################################################
+
+class MinkeConfiguration(Configuration):
+
+    CONF_PATHS = [
+        '/etc/minke.yaml',
+        os.path.expanduser('~/.minke.yaml'),
+        os.path.abspath('conf/minke.yaml'),
+        os.path.abspath('minke.yaml'),
+        os.path.abspath('.minke.yaml'),
+    ]
+
+    debug      = False
+    testing    = False
+
+    # Notification parameters
+    notify     = NotifyConfiguration()
+
+
+##########################################################################
+## Generate Site Settings
+##########################################################################
+
+settings = MinkeConfiguration.load()
+
+if __name__ == '__main__':
+    print(settings)

minke/console/__init__.py

@@ -0,0 +1,21 @@
+# minke.console
+# Implements the console utility called "sei"
+#
+# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
+# Created:  Sat Aug 06 16:15:00 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: __init__.py [] benjamin@bengfort.com $
+
+"""
+Implements the console utility called "sei"
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+from .app import COMMANDS
+from .app import MinkeUtility

minke/console/app.py

@@ -0,0 +1,53 @@
+# minke.console.app
+# Definition of the MinkeUtility app and commands.
+#
+# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
+# Created:  Sat Aug 06 16:17:05 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: app.py [] benjamin@bengfort.com $
+
+"""
+Definition of the MinkeUtility app and commands.
+http://bbengfort.github.io/tutorials/2016/01/23/console-utility-commis.html
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+from commis import color
+from commis import ConsoleProgram
+
+from minke.version import get_version
+from minke.console.commands import *
+
+##########################################################################
+## Utility Definition
+##########################################################################
+
+DESCRIPTION = "Run modeling and adminstrative commands on the Baleen corpus"
+EPILOG      = "If there are any bugs or concerns, submit an issue on GitHub"
+COMMANDS    = [
+    SampleCommand,
+    DescribeCommand,
+]
+
+##########################################################################
+## The Minke CLI Utility
+##########################################################################
+
+class MinkeUtility(ConsoleProgram):
+
+    description = color.format(DESCRIPTION, color.CYAN)
+    epilog      = color.format(EPILOG, color.MAGENTA)
+    version     = color.format("minke (sei) v{}", color.CYAN, get_version())
+
+    @classmethod
+    def load(klass, commands=COMMANDS):
+        utility = klass()
+        for command in commands:
+            utility.register(command)
+        return utility

minke/console/commands/__init__.py

@@ -0,0 +1,21 @@
+# minke.console.commands
+# Commands for the Minke CLI utility
+#
+# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
+# Created:  Sat Aug 06 16:29:54 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: __init__.py [] benjamin@bengfort.com $
+
+"""
+Commands for the Minke CLI utility
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+from .sample import SampleCommand
+from .describe import DescribeCommand

minke/console/commands/describe.py

@@ -0,0 +1,73 @@
+# minke.console.commands.describe
+# Command to describe a corpus for monitoring.
+#
+# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
+# Created:  Sat Aug 06 16:45:51 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: describe.py [] benjamin@bengfort.com $
+
+"""
+Command to describe a corpus for monitoring.
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import os
+
+from commis import Command
+from operator import itemgetter
+from minke.corpus import BaleenCorpusReader
+from minke.utils.humanize import filesize
+
+
+##########################################################################
+## Command
+##########################################################################
+
+class DescribeCommand(Command):
+
+    name = "describe"
+    help = "describe corpus properties for monitoring"
+    args = {
+        'corpus': {
+            'nargs': 1,
+            'help': 'the path to the corpus to describe',
+        }
+    }
+
+    def handle(self, args):
+        """
+        Handle the describe command.
+        """
+        self.corpus = BaleenCorpusReader(args.corpus[0])
+        return self.disk_usage()
+
+    def disk_usage(self):
+        """
+        Returns disk usage properties of the corpus.
+        """
+        output = []
+
+        # Global disk usage statement
+        output.append(
+            "{} documents in {} categories ({})".format(
+                len(self.corpus.fileids()), len(self.corpus.categories()),
+                filesize(sum(s[1] for s in self.corpus.sizes()))
+            )
+        )
+
+        # Per category usage statement
+        for cat in self.corpus.categories():
+            csize = sum(size[1] for size in self.corpus.sizes(categories=cat))
+            output.append(
+                "  - {}: {} ({})".format(
+                    cat, len(self.corpus.fileids(categories=cat)), filesize(csize)
+                )
+            )
+
+        return "\n".join(output)