Merge pull request #1654 from shanzi/md

Add a proxy to use the API entries of nine.buildbot.net

master/buildbot/scripts/processwwwindex.py

@@ -0,0 +1,67 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+
+import os
+import jinja2
+
+from buildbot.test.fake import fakemaster
+from buildbot.util import in_reactor
+from buildbot.util import json
+from twisted.internet import defer
+from buildbot.www.config import IndexResource
+from buildbot.www.service import WWWService
+
+
+@in_reactor
+@defer.inlineCallbacks
+def processwwwindex(config):
+    master = yield fakemaster.make_master()
+    master_service = WWWService(master)
+
+    if not config.get('index-file'):
+        print "Path to the index.html file is required with option --index-file or -i"
+        defer.returnValue(1)
+    path = config.get('index-file')
+    if not os.path.isfile(path):
+        print "Invalid path to index.html"
+        defer.returnValue(2)
+
+    main_dir = os.path.dirname(path)
+
+    for name in master_service.apps.names:
+        if name != 'base':
+            pluginapp = master_service.apps.get(name)
+            try:
+                os.symlink(pluginapp.static_dir, os.path.join(main_dir, name))
+            except OSError:
+                pass
+
+    plugins = dict((k, {}) for k in master_service.apps.names if k != "base")
+
+    fakeconfig = {"user": {"anonymous": True}}
+    fakeconfig['buildbotURL'] = master.config.buildbotURL
+    fakeconfig['title'] = master.config.title
+    fakeconfig['titleURL'] = master.config.titleURL
+    fakeconfig['multiMaster'] = master.config.multiMaster
+    fakeconfig['versions'] = IndexResource.getEnvironmentVersions()
+    fakeconfig['plugins'] = plugins
+    outputstr = ''
+    with open(path) as indexfile:
+        template = jinja2.Template(indexfile.read())
+        outputstr = template.render(configjson=json.dumps(fakeconfig), config=fakeconfig)
+    with open(path, 'w') as indexfile:
+        indexfile.write(outputstr)
+    defer.returnValue(0)

master/buildbot/scripts/runner.py

@@ -642,6 +642,16 @@ def getSynopsis(self):
         return "Usage:   buildbot dataspec [options]"
 
 
+class ProcessWWWIndexOption(base.BasedirMixin, base.SubcommandOptions):
+    """This command is used with the front end's proxy task. It enables to run the front end
+    without the backend server runing in the background"""
+
+    subcommandFunction = "buildbot.scripts.processwwwindex.processwwwindex"
+    optParameters = [
+        ['index-file', 'i', None, "Path to the index.html file to be processed"],
+    ]
+
+
 class Options(usage.Options):
     synopsis = "Usage:    buildbot <command> [command options]"
 
@@ -671,7 +681,11 @@ class Options(usage.Options):
         ['user', None, UserOptions,
          "Manage users in buildbot's database"],
         ['dataspec', None, DataSpecOption,
-         "Output data api spec"]
+         "Output data api spec"],
+        ['processwwwindex', None, ProcessWWWIndexOption,
+         "Process the index.html to enable the front end package working without backend. "
+         "This is a command to work with the frontend's proxy task."
+         ]
     ]
 
     def opt_version(self):

master/buildbot/test/unit/test_scripts_processwwwindex.py

@@ -0,0 +1,61 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+
+import tempfile
+
+from twisted.trial import unittest
+
+from buildbot.util import json
+from buildbot.scripts import processwwwindex
+
+
+class TestUsersClient(unittest.TestCase):
+
+    def setUp(self):
+        # un-do the effects of @in_reactor
+        self.patch(processwwwindex, 'processwwwindex', processwwwindex.processwwwindex._orig)
+
+    def test_no_input_file(self):
+        d = processwwwindex.processwwwindex({})
+
+        def check(ret):
+            self.assertEqual(ret, 1)
+        d.addCallback(check)
+        return d
+
+    def test_invalid_input_file(self):
+        d = processwwwindex.processwwwindex({'index-file': '/some/no/where'})
+
+        def check(ret):
+            self.assertEqual(ret, 2)
+        d.addCallback(check)
+        return d
+
+    def test_output_config(self):
+        tmpf = tempfile.NamedTemporaryFile(suffix=".html")
+        with open(tmpf.name, 'w') as f:
+            f.write('{{ configjson|safe }}')
+
+        d = processwwwindex.processwwwindex({'index-file': tmpf.name})
+
+        def check(ret):
+            self.assertEqual(ret, 0)
+            with open(tmpf.name) as f:
+                config = json.loads(f.read())
+                self.assertTrue(isinstance(config, dict))
+
+        d.addCallback(check)
+        return d

master/buildbot/www/config.py

@@ -44,7 +44,8 @@ def reconfigResource(self, new_config):
     def render_GET(self, request):
         return self.asyncRenderHelper(request, self.renderIndex)
 
-    def getEnvironmentVersions(self):
+    @staticmethod
+    def getEnvironmentVersions():
         import sys
         import twisted
         from buildbot import version as bbversion

www/md_base/guanlecoja/config.coffee

@@ -88,4 +88,47 @@ gulp.task 'icons', ->
         ))
         .pipe(gulp.dest(path.join(config.dir.build, 'icons')))
 
+gulp.task 'processindex', ['index'], ->
+    indexpath = path.join(config.dir.build, 'index.html')
+    gulp.src ""
+        .pipe shell("buildbot processwwwindex -i '#{indexpath}'")
+
+gulp.task 'proxy', ['processindex'], ->
+    # this is a task for developing, it proxy api request to http://nine.buildbot.net
+    argv = require('minimist')(process.argv)
+    argv.host?= 'nine.buildbot.net'
+    argv.port?= 8020
+
+    fs = require 'fs'
+    path = require 'path'
+    http = require 'http'
+    httpProxy = require 'http-proxy'
+    proxy = httpProxy.createProxyServer({})
+    proxy.on 'proxyReq', (proxyReq, req, res, options) ->
+        delete proxyReq.removeHeader('Origin')
+        delete proxyReq.removeHeader('Referer')
+    proxy.on 'proxyRes', (proxyRes, req, res) ->
+        proxyRes.headers['Access-Control-Allow-Origin'] = '*'
+        console.log "[Proxy] #{req.method} #{req.url}"
+
+    server = http.createServer (req, res) ->
+        if req.url.match /^\/(api|sse)/
+            proxy.web req, res, {target: 'http://' + argv.host}
+        else if req.url.match /^\/ws/
+            proxy.ws req, res, {target: 'ws://' + argv.host}
+        else
+            filepath = config.dir.build + req.url.split('?')[0]
+            if fs.existsSync(filepath) and fs.lstatSync(filepath).isDirectory()
+                filepath = path.join(filepath, 'index.html')
+            fs.readFile filepath, (err, data) ->
+                if err
+                    res.writeHead(404)
+                    res.end(JSON.stringify(err))
+                else
+                    res.writeHead(200)
+                    res.end(data)
+
+    server.listen parseInt(argv.port)
+    console.log "[Proxy] server listening on port #{argv.port}"
+
 module.exports = config

www/md_base/package.json

@@ -3,7 +3,9 @@
   "devDependencies": {
     "guanlecoja": "latest",
     "gulp-shell": "^0.4.0",
-    "gulp-svg-symbols": "^0.3.1"
+    "gulp-svg-symbols": "^0.3.1",
+    "http-proxy": "^1.11.1",
+    "minimist": "^1.1.1"
   },
   "name": "md-base",
   "dependencies": {},

www/md_base/src/app/index.jade

@@ -25,5 +25,11 @@ html(ng-app="app", ng-controller="appController as app")
           h1 Hello buildbot!
 
       script(src="scripts.js?_#{(new Date()).getTime()}")
+      | {% for app in config.plugins %}
+      script(src="{{app}}/scripts.js")
+      link(rel='stylesheet', href='{{app}}/styles.css')
+      script
+        | angular.module('app').requires.push('{{app}}')
+      | {% endfor %}
       script
         | angular.module("app").constant("config", {{configjson|safe}})