Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Hopefully helps people in setting r3 up. Should fix #2

  • Loading branch information...
commit b7637bed7321c3a1f25e161a49c3a5a56fed4f28 1 parent 56e5cc8
Bernardo Heynemann authored August 13, 2012
2  Makefile
... ...
@@ -1,6 +1,6 @@
1 1
 # %%%%%%%%%%%%%% SERVICE %%%%%%%%%%%%%%
2 2
 run:
3  
-	@PYTHONPATH=$$PYTHONPATH:.:./test python r3/app/server.py --redis-port=7778 --redis-pass=r3 --config-file="./test/app_config.py"
  3
+	@PYTHONPATH=$$PYTHONPATH:.:./test python r3/app/server.py --redis-port=7778 --redis-pass=r3 --config-file="./test/app_config.py" --debug
4 4
 
5 5
 
6 6
 # %%%%%%%%%%%%%% WORKER %%%%%%%%%%%%%%
30  r3/app/app.py
@@ -6,39 +6,51 @@
6 6
 
7 7
 from r3.app.handlers.healthcheck import HealthcheckHandler
8 8
 from r3.app.handlers.stream import StreamHandler
  9
+from r3.app.handlers.index import IndexHandler
9 10
 from r3.app.utils import kls_import
10 11
 
11 12
 class R3ServiceApp(tornado.web.Application):
12 13
 
13  
-    def __init__(self, redis, config, log_level):
  14
+    def __init__(self, redis, config, log_level, debug, show_index_page):
14 15
         self.redis = redis
15 16
         self.log_level = log_level
16 17
         self.config = config
  18
+        self.debug = debug
17 19
 
18 20
         handlers = [
19 21
             (r'/healthcheck', HealthcheckHandler),
20  
-            (r'/stream/(?P<job_key>.+)/?', StreamHandler),
21 22
         ]
22 23
 
  24
+        if show_index_page:
  25
+            handlers.append(
  26
+                (r'/', IndexHandler)
  27
+            )
  28
+
  29
+        handlers.append(
  30
+            (r'/stream/(?P<job_key>.+)/?', StreamHandler),
  31
+        )
  32
+
23 33
         self.redis.delete('r3::mappers')
24 34
 
25 35
         self.load_input_streams()
26 36
         self.load_reducers()
27 37
 
28  
-        super(R3ServiceApp, self).__init__(handlers)
  38
+        super(R3ServiceApp, self).__init__(handlers, debug=debug)
29 39
 
30 40
     def load_input_streams(self):
31 41
         self.input_streams = {}
32 42
 
33  
-        for stream_class in self.config.INPUT_STREAMS:
34  
-            stream = kls_import(stream_class)
35  
-            self.input_streams[stream.job_type] = stream()
  43
+        if hasattr(self.config, 'INPUT_STREAMS'):
  44
+            for stream_class in self.config.INPUT_STREAMS:
  45
+                stream = kls_import(stream_class)
  46
+                self.input_streams[stream.job_type] = stream()
36 47
 
37 48
     def load_reducers(self):
38 49
         self.reducers = {}
39 50
 
40  
-        for reducer_class in self.config.REDUCERS:
41  
-            reducer = kls_import(reducer_class)
42  
-            self.reducers[reducer.job_type] = reducer()
  51
+        if hasattr(self.config, 'REDUCERS'):
  52
+            for reducer_class in self.config.REDUCERS:
  53
+                reducer = kls_import(reducer_class)
  54
+                self.reducers[reducer.job_type] = reducer()
43 55
 
44 56
 
24  r3/app/handlers/index.py
... ...
@@ -0,0 +1,24 @@
  1
+#!/usr/bin/python
  2
+# -*- coding: utf-8 -*-
  3
+
  4
+from r3.app.handlers import BaseHandler
  5
+from r3.app.keys import MAPPERS_KEY
  6
+from r3.version import __version__
  7
+
  8
+class IndexHandler(BaseHandler):
  9
+    def get(self):
  10
+        has_reducers = len(self.application.reducers.keys()) > 0
  11
+
  12
+        self.render(
  13
+            "../templates/index.html", 
  14
+            title="",
  15
+            r3_version=__version__,
  16
+            input_streams=self.application.input_streams.keys(),
  17
+            has_reducers=has_reducers,
  18
+            mappers=self.get_mappers()
  19
+        )
  20
+
  21
+    def get_mappers(self):
  22
+        return self.redis.smembers(MAPPERS_KEY)
  23
+
  24
+
4  r3/app/server.py
@@ -20,6 +20,8 @@ def main(arguments=None):
20 20
     parser.add_argument('-b', '--bind', type=str, default='0.0.0.0', help='the ip that r³ will bind to')
21 21
     parser.add_argument('-p', '--port', type=int, default=9999, help='the port that r³ will bind to')
22 22
     parser.add_argument('-l', '--loglevel', type=str, default='warning', help='the log level that r³ will run under')
  23
+    parser.add_argument('-i', '--hide-index-page', action='store_true', default=False, help='indicates whether r³ app should show the help page')
  24
+    parser.add_argument('-d', '--debug', action='store_true', default=False, help='indicates whether r³ app should run in debug mode')
23 25
     parser.add_argument('--redis-host', type=str, default='0.0.0.0', help='the ip that r³ will use to connect to redis')
24 26
     parser.add_argument('--redis-port', type=int, default=6379, help='the port that r³ will use to connect to redis')
25 27
     parser.add_argument('--redis-db', type=int, default=0, help='the database that r³ will use to connect to redis')
@@ -34,7 +36,7 @@ def main(arguments=None):
34 36
 
35 37
     logging.basicConfig(level=getattr(logging, args.loglevel.upper()))
36 38
 
37  
-    application = R3ServiceApp(redis=c, config=cfg, log_level=args.loglevel.upper())
  39
+    application = R3ServiceApp(redis=c, config=cfg, log_level=args.loglevel.upper(), debug=args.debug, show_index_page=not args.hide_index_page)
38 40
 
39 41
     server = HTTPServer(application)
40 42
     server.bind(args.port, args.bind)
254  r3/app/templates/index.html
... ...
@@ -0,0 +1,254 @@
  1
+<!DOCTYPE HTML>
  2
+<html>
  3
+   <head>
  4
+        <title>r³ - map reduce service</title>
  5
+        <style>
  6
+            /* This is here because of the monstrous base64 */
  7
+            header a h1 {
  8
+                background: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKIAAAA+CAMAAACfiDARAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyJpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMC1jMDYwIDYxLjEzNDc3NywgMjAxMC8wMi8xMi0xNzozMjowMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNSBNYWNpbnRvc2giIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6RUZGMjM3Q0FDM0E5MTFFMUFDMTNCQTVFOUI0MTVDQUYiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6RUZGMjM3Q0JDM0E5MTFFMUFDMTNCQTVFOUI0MTVDQUYiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDpFRkYyMzdDOEMzQTkxMUUxQUMxM0JBNUU5QjQxNUNBRiIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDpFRkYyMzdDOUMzQTkxMUUxQUMxM0JBNUU5QjQxNUNBRiIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pv5Mc2IAAABmUExURXqdyKa92liEuvT3+mOMvoqdtenv9m+Vw5Ct0d3m8bHF3rzO45u21sfW6KC52IWlzOfu9fP2+nyfyZSw01mFurjL4YioznGWxNDc66zC3Nvl8GWNv8TT5nKQtWWJtdLe7P///018tRm+KqsAAAQtSURBVHja7Jhrd+MoDIa5LRiwnaZJOp12dhH//0+OEL6mrseeuHH3nPAhh4BsHgSSX2Dx78t/8S6F3fDsPw/EB+L/DjGEyGAwFFNtBVs5fLAXTCxGPL/Udf28BaLiqv8PvKmk1gnE8XxmX306QipPL7cjjkqHmMpNiCek+1n/QsiXpYguFIVGn3kJmqfVMj5403qR+VAVWIXAFeOCh+xF4VIzwhJNUTAPHieBzaGYRzwf4e3ckC5FDBAAgpHkfSvQH5p7kHkvFjgyjm4C6GCQU4e8F2Wy8a1zkcuCDdFgs5NWzCHWAD+o8qutLEEE7QpkUJFJHFKnJS4hI0pH66qIhUPVhktFNocOMS+0Tc0G+BziBY4d6/NyRBEFQFo3B7bZVI0Xk1spWjOiahEN7U8/RmS5ubITiObAOdm+wmtu+bnGiyEN3JYmFJpRHciKmyZcqGuQdNBmhMiBpxKAKd4WSgqCa3r3aNynVXuREG2gMkaMzFncqWIhYn5FYP2Mk0OZbP4MRv3xuiKiCVHlhaYdRhPXferGcC6vEbNNu9A2G5fQpO8xYtH96VMYluNzXIUYddp1QoNXxGpyuBiMiLzKV4jJJlmCo0pGzI8K6UeIfX2MCMfTOsQDYJ6RIFWsJA5mm1G1NZjsUkRXTAwQycbiw9JEk40xjunRINUQUcgJxBjfT08Ap1WI8ZB2tMbIEBVWXLO9TGqV6ByPow29mHIoGlN/MsZlgDQZfAcbLXQFk4iUw49rP4CmlQpqKAkMY83+upINpt13YtCg6HeA6OAzRMyQU4nxXmKMwWTBrueeq/6miO8AdZzL3fshUr7MGbv5upync/c+iNKZkRajQD5fen9uoxdvQKzEqPMN4K2uL6hrL/GbIFbXgvGS2491/CaIUnzofl97dplX3VlYizIE1/RVbBWi2+AEOKu68wda2KSm0R8Fng9IbS9HZJsgzqjujOjTahkolSypj61AjNsgfq66G5lDq+Udl/SEdMsR5TaIM6qbELsDaNCkpHVYjhi2RJxW3WNEmW3cHoifq24BvBG2xlR2ddLZDnFadSeyAvei9FlNF4TP2tncGXFCdQs8tMdDCvISDy/Ko4LWFsNe2328OKG6YyGT4s6nlqy9VUijqnsjzqhu0UntrtYZ7YL4JWLsgfhAfCA+EP+gutvr60ZrUxrEChf7IV6r7u7KMGltrqXBinRU2Q9xrLo7RLouRPEQnU4VHfZEHKruHjHfIhoTocwqTey40EPV3SMqCdYV6bzlk9b2y09KX6y6e0QMlwrlWcHaLrMrYq+6++trKgcrG9Edd43ooerur6/zMQDPMtrns9VyxBty6hLV3V1fF1DiRkS+JLqFW+HLr0HsVXd/fU2XwAHjuEzyu4y7IX5U3WygutWV/N4XccNv9APxgfhAvAfiv/dB/C3AAEqhoW7VYnBkAAAAAElFTkSuQmCC') no-repeat;
  9
+                width: 162px;
  10
+                height: 62px;
  11
+                margin-top: 20px;
  12
+                margin-left: 40px;
  13
+                text-indent: -999999px;
  14
+            }
  15
+
  16
+
  17
+
  18
+            /* http://meyerweb.com/eric/tools/css/reset/ 
  19
+               v2.0 | 20110126
  20
+               License: none (public domain)
  21
+            */
  22
+
  23
+            html, body, div, span, applet, object, iframe,
  24
+            h1, h2, h3, h4, h5, h6, p, blockquote, pre,
  25
+            a, abbr, acronym, address, big, cite, code,
  26
+            del, dfn, em, img, ins, kbd, q, s, samp,
  27
+            small, strike, strong, sub, sup, tt, var,
  28
+            b, u, i, center,
  29
+            dl, dt, dd, ol, ul, li,
  30
+            fieldset, form, label, legend,
  31
+            table, caption, tbody, tfoot, thead, tr, th, td,
  32
+            article, aside, canvas, details, embed, 
  33
+            figure, figcaption, footer, header, hgroup, 
  34
+            menu, nav, output, ruby, section, summary,
  35
+            time, mark, audio, video {
  36
+                margin: 0;
  37
+                padding: 0;
  38
+                border: 0;
  39
+                font-size: 100%;
  40
+                font: inherit;
  41
+                vertical-align: baseline;
  42
+            }
  43
+            /* HTML5 display-role reset for older browsers */
  44
+            article, aside, details, figcaption, figure, 
  45
+            footer, header, hgroup, menu, nav, section {
  46
+                display: block;
  47
+            }
  48
+            body {
  49
+                line-height: 1;
  50
+            }
  51
+            ol, ul {
  52
+                list-style: none;
  53
+            }
  54
+            blockquote, q {
  55
+                quotes: none;
  56
+            }
  57
+            blockquote:before, blockquote:after,
  58
+            q:before, q:after {
  59
+                content: '';
  60
+                content: none;
  61
+            }
  62
+            table {
  63
+                border-collapse: collapse;
  64
+                border-spacing: 0;
  65
+            }
  66
+
  67
+            ul {
  68
+                list-style: none;
  69
+            }
  70
+
  71
+            body {
  72
+                font-family: arial;
  73
+                background-color: #4d7cb5;
  74
+            }
  75
+
  76
+            .container {
  77
+                width: 940px;
  78
+                margin: 0 auto;
  79
+            }
  80
+
  81
+            header {
  82
+                height: 102px;
  83
+                overflow: hidden;
  84
+            }
  85
+
  86
+            header a {
  87
+                display: block;
  88
+                float: left;
  89
+            }
  90
+
  91
+            footer {
  92
+                width: 100%;
  93
+                height: 100%;
  94
+                background-color: #4d7cb5;
  95
+                text-align: center;
  96
+                overflow: hidden;
  97
+                display: table;
  98
+            }
  99
+
  100
+            footer .cell {
  101
+                display: table-cell;
  102
+                vertical-align: middle;
  103
+                text-align: center;
  104
+            }
  105
+
  106
+            footer a {
  107
+                color: #fff;
  108
+                font-weight: bold;
  109
+                text-decoration: none;
  110
+                font-size: 11px;
  111
+                display: block;
  112
+                margin: 20px 0;
  113
+            }
  114
+
  115
+            footer a:hover {
  116
+                text-decoration: underline;
  117
+            }
  118
+
  119
+            .body {
  120
+                background-color: #fff;
  121
+                padding-bottom: 20px;
  122
+                width: 100%;
  123
+            }
  124
+
  125
+            .body .container .contents {
  126
+                padding-left: 40px;
  127
+                padding-top: 20px;
  128
+            }
  129
+
  130
+            .body .container .contents h1 {
  131
+                font-size: 32px;
  132
+                font-weight: bold;
  133
+                color: #4d7cb5;
  134
+            }
  135
+
  136
+            .body .container .contents .section {
  137
+                margin-top: 20px;
  138
+                line-height: 21px;
  139
+                font-size: 13px;
  140
+                color: #333;
  141
+            }
  142
+
  143
+            .body .container .contents .section h2 {
  144
+                font-size: 24px;
  145
+                font-weight: bold;
  146
+                margin-bottom: 10px;
  147
+            }
  148
+
  149
+            .body .container .contents .section h2.warning {
  150
+                color: #a00000;
  151
+            }
  152
+
  153
+            .body .container .contents .section .code {
  154
+                margin: 10px;
  155
+            }
  156
+
  157
+            .body .container .contents .available-mappers ul {
  158
+                overflow: hidden;
  159
+            }
  160
+
  161
+            .body .container .contents .available-mappers li {
  162
+                font-size: 13px;
  163
+                font-weight: bold;
  164
+                color: #666;
  165
+                float: left;
  166
+                width: 225px;
  167
+            }
  168
+        </style>
  169
+    </head>
  170
+    <body>
  171
+        <header>
  172
+            <div class="container">
  173
+                <a href="/" alt="overview"><h1>r³ - map reduce service</h1></a>
  174
+            </div>
  175
+        </header>
  176
+
  177
+        <div class="body">
  178
+            <div class="container">
  179
+                <div class="contents">
  180
+                    <h1>Welcome to r³</h1>
  181
+
  182
+                    {% if not input_streams %}
  183
+                    <div class="stream section">
  184
+                        <h2 class="warning">It appears you didn't setup any input streams!</h2>
  185
+                        <p>Input Streams are classes that generate units-of-work that your mappers will work on.</p>
  186
+                        <p>Creating them is as simple as creating a class that has a process method and a job_type argument:</p>
  187
+                        <div class="code"><script src="https://gist.github.com/3341367.js"></script></div>
  188
+                        <p>After you create your input stream, just add it to a config.py file:</p>
  189
+                        <div class="code"><script src="https://gist.github.com/3341383.js?file=config.py"></script></div>
  190
+                        <p>Then pass the file path as an argument to r3-app like this:</p>
  191
+                        <div class="code"><script src="https://gist.github.com/3341383.js?file=r3.sh"></script></div>
  192
+                        <p>For more information check the <a href="http://heynemann.github.com/r3/">documentation online</a>.</p>
  193
+                    </div>
  194
+                    {% end %}
  195
+
  196
+                    {% if not mappers %}
  197
+                    <div class="mappers section">
  198
+                        <h2 class="warning">It appears you didn't setup any mappers!</h2>
  199
+                        <p>Setting mappers to run your map/reduce tasks is an integral part of r³ and is as simple as creating a class that inherits from Mapper:</p>
  200
+                        <div class="code"><script src="https://gist.github.com/3341253.js?file=mapper.py"></script></div>
  201
+                        <p>Running the mappers is pretty simple as well. Say you want to run four different mappers:</p>
  202
+                        <div class="code"><script src="https://gist.github.com/3341253.js?file=mapper.sh"></script></div>
  203
+                        <p>For more information check the <a href="http://heynemann.github.com/r3/">documentation online</a>.</p>
  204
+                    </div>
  205
+                    {% end %}
  206
+
  207
+                    {% if not has_reducers %}
  208
+                    <div class="reducers section">
  209
+                        <h2 class="warning">It appears you didn't setup any reducers!</h2>
  210
+                        <p>Reducers are the classes that get the mapped units-of-work generated by your mappers and process them into a single coherent result.</p>
  211
+                        <p>Creating them is as simple as creating a class that has a reduce method and a job_type argument:</p>
  212
+                        <div class="code"><script src="https://gist.github.com/3341423.js"></script></div>
  213
+                        <p>After you create your input stream, just add it to a config.py file:</p>
  214
+                        <div class="code"><script src="https://gist.github.com/3341383.js?file=config.py"></script></div>
  215
+                        <p>Then pass the file path as an argument to r3-app like this:</p>
  216
+                        <div class="code"><script src="https://gist.github.com/3341383.js?file=r3.sh"></script></div>
  217
+                        <p>For more information check the <a href="http://heynemann.github.com/r3/">documentation online</a>.</p>
  218
+                    </div>
  219
+                    {% end %}
  220
+
  221
+                    {% if mappers %}
  222
+                    <div class="available-mappers section">
  223
+                        <h2>Available Mappers ({{ len(mappers) }})</h2>
  224
+                        <ul>
  225
+                            {% for mapper in mappers %}
  226
+                            <li>* {{ mapper }}</li>
  227
+                            {% end %}
  228
+                        </ul>
  229
+                    </div>
  230
+                    {% end %}
  231
+
  232
+                    {% if input_streams %}
  233
+                    <div class="available-mappers section">
  234
+                        <h2>Available Streams</h2>
  235
+                        <p>Please be advised that the link below may not work if your input stream requires additional arguments in it's URL.</p>
  236
+                        <ul>
  237
+                            {% for stream in input_streams %}
  238
+                            <li><a href="/stream/{{ stream }}">{{ stream }}</a></li>
  239
+                            {% end %}
  240
+                        </ul>
  241
+                    </div>
  242
+                    {% end %}
  243
+
  244
+                </div>
  245
+            </div>
  246
+        </div>
  247
+
  248
+        <footer>
  249
+            <div class="cell">
  250
+                <a href="http://github.com/heynemann/r3" alt="r3 map reduce service">powered by r³ version {{ r3_version }}</a>
  251
+            </div>
  252
+        </footer>
  253
+    </body>
  254
+</html>

0 notes on commit b7637be

Please sign in to comment.
Something went wrong with that request. Please try again.