Building out structure. Added webserver.

.gitignore

@@ -0,0 +1,8 @@
+*.pyc
+/bin/
+/lib/
+/include/
+/.Python
+/build/
+/dist/
+/mapreduce.egg-info/

README.rst

@@ -0,0 +1 @@
+mr/resources/README.rst

dev/start_dev.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python2.7
+
+import sys
+import os
+dev_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.insert(0, dev_path)
+
+os.environ['DEBUG'] = '1'
+
+import mr.app.main
+
+mr.app.main.app.run()

mr/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.2.0'

mr/app/main.py

@@ -0,0 +1,21 @@
+import sys
+import os
+dev_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), 
+            '..', 
+            'resources', 
+            'templates'))
+
+sys.path.insert(0, dev_path)
+
+import flask
+
+import mr.config
+import mr.views.job
+import mr.views.index
+
+app = flask.Flask(__name__)
+app.debug = mr.config.IS_DEBUG
+
+app.register_blueprint(mr.views.index.index_bp)
+app.register_blueprint(mr.views.job.job_bp)

mr/config/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+IS_DEBUG = bool(int(os.environ.get('DEBUG', '0')))

mr/handlers.py

@@ -0,0 +1,25 @@
+class HandlersBase(object):
+    """The base-class of our handler code libraries."""
+
+    def get_handler_list_version(self):
+        """Get an opaque string that describes the set of handlers and their 
+        classifications.
+        """
+
+        raise NotImplementedError()
+
+    def get_handler_classifications(self):
+        """Get a list of handlers and the classifications that they represent.
+        """
+
+        raise NotImplementedError()
+
+    def get_code_handler(self, handler_name):
+        """Return the code for the given handler."""
+
+        raise NotImplementedError()
+
+    def get_code_all_handlers(self):
+        """Return the code for each and every current handler."""
+
+        raise NotImplementedError()

mr/job_handlers.py

@@ -0,0 +1,29 @@
+import mr.handlers
+
+
+class JobHandlersBase(mr.handlers.HandlersBase):
+    """Manages the code for each job handler."""
+
+    def get_handler_list_version(self):
+        """Get an opaque string that describes the set of handlers and their 
+        job classifications.
+        """
+
+        raise NotImplementedError()
+
+    def get_handler_classifications(self):
+        """Get a list of handlers and the job classifications that they 
+        represent.
+        """
+
+        raise NotImplementedError()
+
+    def get_code_handler(self, handler_name):
+        """Return the code for the given handler."""
+
+        raise NotImplementedError()
+
+    def get_code_all_handlers(self):
+        """Return the code for each and every current handler."""
+
+        raise NotImplementedError()

mr/kv.py

@@ -0,0 +1,6 @@
+class KvAdapter(object):
+    def set(self, key, value):
+        raise NotImplementedError()
+
+    def get(self, key):
+        raise NotImplementedError()

mr/mappings.py

@@ -0,0 +1,25 @@
+class MappingsBase(object):
+    """Manages the mappings of which jobs/steps are mapped to (and reduced 
+    from) which jobs/steps.
+    """
+
+    def set_job_to_step_mapping(self, job, step):
+        raise NotImplementedError()
+
+    def set_step_to_step_mapping(self, step_from, step_to):
+        raise NotImplementedError()
+
+    def set_step_to_step_reduction(self, step_from, step_to):
+        raise NotImplementedError()
+
+    def set_step_to_job_reduction(self, step, job):
+        raise NotImplementedError()
+
+    def get_mapping_from_job(self, job_name):
+        raise NotImplementedError()
+
+    def get_mapping_from_step(self, step_name):
+        raise NotImplementedError()
+
+    def get_reduction_from_step(self, step_name):
+        raise NotImplementedError()

mr/resources/README.rst

@@ -0,0 +1,10 @@
+
+
+Vocabulary
+----------
+
+- Job: Represents the original request received by the MR.
+- Step: Represents one or more handlers that a job or another step can map 
+        into, or be reduced from.
+- Response: Represents the final step of reduction, and the result to the 
+            original job.

mr/resources/data/gunicorn_conf_dev.py

@@ -0,0 +1,12 @@
+debug = 'true'
+daemon = 'false'
+
+bind = 'unix:/tmp/mr.gunicorn.sock'
+
+# Until our packages become smaller (currently 161M), this is a safe value.
+timeout = 120
+
+errorlog = '-'
+loglevel = 'debug'
+
+worker_class = 'gevent'

mr/resources/requirements.txt

@@ -0,0 +1,6 @@
+Flask==0.10.1
+Jinja2==2.7.3
+MarkupSafe==0.23
+Werkzeug==0.9.6
+itsdangerous==0.24
+wsgiref==0.1.2

mr/resources/scripts/mr_start_gunicorn_dev

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import sys
+import os.path
+dev_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+sys.path.insert(0, dev_path)
+
+import os.path
+import mr
+root_path = os.path.abspath(os.path.join(os.path.dirname(mr.__file__), '..'))
+
+import os
+os.chdir(root_path)
+
+import subprocess
+
+cmd = ['gunicorn', '-c', 'mr/resources/data/gunicorn_conf_dev.py', 'mr.app.main:app']
+env = { 'DEBUG': '1' }
+
+p = subprocess.Popen(cmd, env=env)
+r = p.wait()
+if r != 0:
+    raise EnvironmentError("Gunicorn launch failed.")

mr/step_handlers.py

@@ -0,0 +1,29 @@
+import mr.handlers
+
+
+class StepHandlersBase(mr.handlers.HandlersBase):
+    """Manages the code for each step handler."""
+
+    def get_handler_list_version(self):
+        """Get an opaque string that describes the set of handlers and their 
+        step classifications.
+        """
+
+        raise NotImplementedError()
+
+    def get_handler_classifications(self):
+        """Get a list of handlers and the step classifications that they 
+        represent.
+        """
+
+        raise NotImplementedError()
+
+    def get_code_handler(self, handler_name):
+        """Return the code for the given handler."""
+
+        raise NotImplementedError()
+
+    def get_code_all_handlers(self):
+        """Return the code for each and every current handler."""
+
+        raise NotImplementedError()

mr/views/index.py

@@ -0,0 +1,9 @@
+import flask
+
+index_bp = flask.Blueprint(
+            '', 
+            __name__)
+
+@index_bp.route('/')
+def index():
+    return "Map me. Reduce me."

mr/views/job.py

@@ -0,0 +1,10 @@
+import flask
+
+job_bp = flask.Blueprint(
+            'job', 
+            __name__,
+            url_prefix='/job')
+
+@job_bp.route('/<classification>')
+def job_submit(classification):
+    return flask.jsonify({ 'job_id': 123 })

setup.py

@@ -0,0 +1,35 @@
+import setuptools
+import os.path
+
+import mr
+
+_APP_PATH = os.path.dirname(mr.__file__)
+
+with open(os.path.join(_APP_PATH, 'resources', 'README.rst')) as f:
+      long_description = f.read()
+
+with open(os.path.join(_APP_PATH, 'resources', 'requirements.txt')) as f:
+      install_requires = list(map(lambda s: s.strip(), f.readlines()))
+
+setuptools.setup(
+      name='mapreduce',
+      version=mr.__version__,
+      description="A Python-based, distributed MapReduce solution.",
+      long_description=long_description,
+      classifiers=[],
+      keywords='',
+      author='Dustin Oprea',
+      author_email='myselfasunder@gmail.com',
+      url='',
+      license='GPL 2',
+      packages=setuptools.find_packages(exclude=['dev']),
+      include_package_data=True,
+      zip_safe=False,
+#      install_requires=install_requires,
+      package_data={
+          'nsq': ['resources/README.rst',
+                  'resources/requirements.txt'],
+      },
+      scripts=[
+      ],
+)