dataService bug fixes and documentation

master/docs/developer/www.rst

@@ -617,6 +617,139 @@ Several methods are added to each "restangularized" objects, aside from get(), p
     Call the control data api.
     This builds up a POST with jsonapi encoded parameters
 
+DataService
+.............
+
+DataService is the future replacement of BuildbotService for accessing the Buildbot data API.
+It has a modern interface for accessing data. It uses IndexedDB for storing cached data as a single data store,
+and LocalStorage for broadcasting events between browser tabs. DataService works in a master/slave architecture.
+The master browser tab is responsible for keeping the requested data up to date in the IndexedDB and notify slaves when a data is ready to be used or it is updated.
+It handles both the Rest API calls and the WebSocket subscriptions globally.
+
+It uses the following libraries:
+
+* Dexie.js (https://github.com/dfahlander/Dexie.js) - Minimalistic IndexedDB API with bulletproof transactions
+* Tabex (https://github.com/nodeca/tabex) - Master election and in browser message bus
+
+The DataService is available as a standalone AngularJS module.
+Installation via bower:
+
+  .. code-block:: none
+
+      bower install buildbot-data --save
+
+Inject the ``bbData`` module to your application:
+
+  .. code-block:: javascript
+
+      angular.module('myApp', ['bbData'])
+
+Methods:
+
+* ``.get(endpoint, [id], [query])``: returns a promise<Collection>, when the promise is resolved, the Collection contains all the requested data
+
+  * call .getArray() on the returned promise to get the updating Collection before it's filled with the initial data
+
+  .. code-block:: coffeescript
+
+      # assign builds to $scope.builds once the Collection is filled
+      dataService.get('builds', builderid: 1).then (builds) ->
+          $scope.builds = builds
+          # load steps for every build
+          builds.forEach (b) -> b.loadSteps()
+
+      # assign builds to $scope.builds before the Collection is filled using the .getArray() function
+      $scope.builds = dataService.get('builds', builderid: 1).getArray()
+
+* ``.getXXX([id], [query])``: returns a promise<Collection>, when the promise is resolved, the Collection contains all the requested data
+
+  * it's higly advised to use these instead of the ``.get('string')`` function
+  * XXX can be the following: Builds, Builders, Buildrequests, Buildsets, Buildslaves, Changes, Changesources, Forceschedulers, Masters, Schedulers, Sourcestamps
+  * call .getArray() on the returned promise to get the updating Collection before it's filled with the initial data
+
+  .. code-block:: coffeescript
+
+      # assign builds to $scope.builds once the Collection is filled
+      dataService.getBuilds(builderid: 1).then (builds) ->
+          $scope.builds = builds
+          # load steps for every build
+          builds.forEach (b) -> b.loadSteps()
+
+      # assign builds to $scope.builds before the Collection is filled using the .getArray() function
+      $scope.builds = dataService.getBuilds(builderid: 1).getArray()
+
+* ``.open(scope)``: returns a DataAccessor, handles bindings, open a new accessor every time you need updating data in a controller
+
+  .. code-block:: coffeescript
+
+      # open a new accessor every time you need updating data in a controller
+      class DemoController extends Controller
+          constructor: ($scope, dataService) ->
+              # automatically closes all the bindings when the $scope is destroyed
+              opened = dataService.open($scope)
+              # alternative syntax:
+              #   opened = dataService.open()
+              #   opened.closeOnDestroy($scope)
+              # closing it manually is also possible:
+              #   opened.close()
+
+              # request new data, it updates automatically
+              @builders = opened.getBuilders(limit: 10, order: '-started_at').getArray()
+
+* ``.control(url, method, [params])``: returns a promise, sends a JSON RPC2 POST request to the server
+
+  .. code-block:: coffeescript
+
+      # open a new accessor every time you need updating data in a controller
+      dataService.control('Forceschedulers/force', 'force').then (response) ->
+          $log.debug(response)
+      , (reason) ->
+          $log.error(reason)
+
+* ``.clearCache()``: clears the IndexedDB tables and reloads the current page
+
+  .. code-block:: coffeescript
+
+      class DemoController extends Controller
+          constructor: (@dataService) ->
+          onClick: -> @dataService.clearCache()
+
+Methods on the Wrapper classes:
+
+* ``.getXXX([id], [query])``: returns a promise<Collection>, when the promise is resolved, the Collection contains all the requested data
+
+  * same as dataService.getXXX, but with relative endpoint
+
+  .. code-block:: coffeescript
+
+      # assign builds to $scope.builds once the Collection is filled
+      dataService.getBuilds(builderid: 1).then (builds) ->
+          $scope.builds = builds
+          # get steps for every build
+          builds.forEach (b) ->
+              b.getSteps().then (steps) ->
+                  # assign completed test to every build
+                  b.complete_steps = steps.map (s) -> s.complete
+
+* ``.loadXXX([id], [query])``: returns a promise<Collection>, the Collection contains all the requested data, it is also assigned to wrapperInstance.xxx
+
+  .. code-block:: coffeescript
+
+      $q (resolve) ->
+          # get builder with id = 1
+          dataService.getBuilders(1).then (builders) ->
+              builders.forEach (builder) ->
+                  # load all builds
+                  builder.loadBuilds().then (builds) ->
+                      builds.forEach (build) ->
+                          # load all buildsteps
+                          build.loadSteps().then -> resolve(builders[0])
+      .then (builder) ->
+          # builder has a builds field, and the builds have a steps field containing the corresponding data
+          $log.debug(builder)
+
+* ``.control(method, params)``: returns a promise, sends a JSON RPC2 POST request to the server
+
 RecentStorage
 .............
 

www/data_module/src/mock/generator.service.coffee

@@ -0,0 +1,31 @@
+# types for generating test data: null, number, string, boolean, timestamp, <array>[], <object>, <objectName in Specification>
+class Generator extends Service
+    self = null
+    constructor: ->
+        self = @
+
+    number: (min = 0, max = 100) ->
+        random = Math.random() * (max - min) + min
+        Math.floor(random)
+
+    ids: {}
+    id: (name = '') ->
+        self.ids[name] ?= 0
+        self.ids[name]++
+
+    boolean: -> Math.random() < 0.5
+
+    timestamp: (after = Date.now()) ->
+        date = new Date(after + self.number(1, 1000000))
+        Math.floor(date.getTime() / 1000)
+
+    string: (length) ->
+        if length? then length++
+        self.number(100, Number.MAX_VALUE).toString(36).substring(0, length)
+
+    array: (fn, args...) ->
+        times = self.number(1, 10)
+        array = []
+        for i in [1..times]
+            array.push fn(args...)
+        return array

www/data_module/src/services/data/data.service.coffee

@@ -2,8 +2,7 @@ class Data extends Provider
     cache: true
     config = null
     constructor: ->
-        config =
-            cache: @cache
+        config = cache: @cache
 
     ### @ngInject ###
     $get: ($log, $injector, $q, $window, Collection, restService, dataUtilsService, tabexService, indexedDBService, SPECIFICATION) ->

www/data_module/src/services/data/wrapper/wrapper.service.coffee

@@ -29,25 +29,33 @@ class Wrapper extends Factory
                 specification = SPECIFICATION[root]
                 match = specification.paths.filter (p) ->
                     replaced = p
-                        .replace ///\w+\:\w+///g, '\\*'
+                        .replace ///\w+\:\w+///g, '(\\*|\\w+|\\d+)'
                     ///^#{replaced}$///.test(pathString)
                 .pop()
                 if not match?
                     parameter = @getId()
-
-                # second last element
-                for e in match.split('/') by -1
-                    if e.indexOf(':') > -1
-                        [fieldType, fieldName] = e.split(':')
-                        parameter = @[fieldName]
+                    $log.debug specification.paths, pathString
+                else
+                    # second last element
+                    for e in match.split('/') by -1
+                        if e.indexOf(':') > -1
+                            [fieldType, fieldName] = e.split(':')
+                            parameter = @[fieldName]
+                            break
 
                 dataService.get(@getEndpoint(), parameter, args...)
 
+            control: (method, params) ->
+                dataService.control(@getEndpoint(), method, params)
+
             # generate endpoint functions for the class
             @generateFunctions: (endpoints) ->
                 endpoints.forEach (e) =>
                     # capitalize endpoint names
                     E = dataUtilsService.capitalize(e)
+                    # adds getXXX functions to the prototype
+                    @::["get#{E}"] = (args...) ->
+                        return @get(e, args...)
                     # adds loadXXX functions to the prototype
                     @::["load#{E}"] = (args...) ->
                         p = @get(e, args...)

www/data_module/src/services/dataUtils/dataUtils.service.coffee

@@ -1,52 +1,68 @@
 class DataUtils extends Service
+    constructor: (SPECIFICATION) ->
+        return new class dataUtilsService
+            # capitalize first word
+            capitalize: (string) ->
+                string[0].toUpperCase() + string[1..].toLowerCase()
 
-    # capitalize first word
-    capitalize: (string) ->
-        string[0].toUpperCase() + string[1..].toLowerCase()
-
-    # returns the type of the endpoint
-    type: (arg) ->
-        a = @copyOrSplit(arg)
-        a = a.filter (e) -> e isnt '*'
-        # if the argument count is even, the last argument is an id
-        if a.length % 2 is 0 then a.pop()
-        a.pop()
-
-    # singularize the type name
-    singularType: (arg) ->
-        @type(arg).replace(/s$/, '')
-
-    socketPath: (arg) ->
-        a = @copyOrSplit(arg)
-        # if the argument count is even, the last argument is an id
-        stars = ['*']
-        # is it odd?
-        if a.length % 2 is 1 then stars.push('*')
-        a.concat(stars).join('/')
-
-    restPath: (arg) ->
-        a = @copyOrSplit(arg)
-        a = a.filter (e) -> e isnt '*'
-        a.join('/')
-
-    endpointPath: (arg) ->
-        # if the argument count is even, the last argument is an id
-        a = @copyOrSplit(arg)
-        a = a.filter (e) -> e isnt '*'
-        # is it even?
-        if a.length % 2 is 0 then a.pop()
-        a.join('/')
-
-    copyOrSplit: (arrayOrString) ->
-        if angular.isArray(arrayOrString)
-            # return a copy
-            arrayOrString[..]
-        else if angular.isString(arrayOrString)
-            # split the string to get an array
-            arrayOrString.split('/')
-        else
-            throw new TypeError("Parameter 'arrayOrString' must be a array or a string, not #{typeof arrayOrString}")
-
-    unWrap: (data, path) ->
-        type = @type(path)
-        data[type]
+            # returns the type of the endpoint
+            type: (arg) ->
+                a = @copyOrSplit(arg)
+                a = a.filter (e) -> e isnt '*'
+                # if the argument count is even, the last argument is an id
+                if a.length % 2 is 0 then a.pop()
+                a.pop()
+
+            # singularize the type name
+            singularType: (arg) ->
+                @type(arg).replace(/s$/, '')
+
+            socketPath: (arg) ->
+                a = @copyOrSplit(arg)
+                # if the argument count is even, the last argument is an id
+                stars = ['*']
+                # is it odd?
+                if a.length % 2 is 1 then stars.push('*')
+                a.concat(stars).join('/')
+
+            restPath: (arg) ->
+                a = @copyOrSplit(arg)
+                a = a.filter (e) -> e isnt '*'
+                a.join('/')
+
+            endpointPath: (arg) ->
+                # if the argument count is even, the last argument is an id
+                a = @copyOrSplit(arg)
+                a = a.filter (e) -> e isnt '*'
+                # is it even?
+                if a.length % 2 is 0 then a.pop()
+                a.join('/')
+
+            copyOrSplit: (arrayOrString) ->
+                if angular.isArray(arrayOrString)
+                    # return a copy
+                    arrayOrString[..]
+                else if angular.isString(arrayOrString)
+                    # split the string to get an array
+                    arrayOrString.split('/')
+                else
+                    throw new TypeError("Parameter 'arrayOrString' must be a array or a string, not #{typeof arrayOrString}")
+
+            unWrap: (data, path) ->
+                type = @type(path)
+                type = SPECIFICATION[type]?.restField or type
+                data[type]
+
+            parse: (object) ->
+                for k, v of object
+                    try
+                        object[k] = angular.fromJson(v)
+                    catch error then # ignore
+                return object
+
+            numberOrString: (str = null) ->
+                # if already a number
+                if angular.isNumber(str) then return str
+                # else parse string to integer
+                number = parseInt str, 10
+                if !isNaN(number) then number else str

www/data_module/src/services/dataUtils/dataUtils.service.spec.coffee

@@ -88,3 +88,30 @@ describe 'Data utils service', ->
 
             result = dataUtilsService.unWrap(data, 'bnm/1/asd/2')
             expect(result).toBe(data.asd)
+
+    describe 'parse(object)', ->
+
+        it 'should parse fields from JSON', ->
+            test =
+                a: 1
+                b: 'asd3'
+                c: angular.toJson(['a', 1, 2])
+                d: angular.toJson({asd: [], bsd: {}})
+
+            copy = angular.copy(test)
+            copy.c = angular.toJson(copy.c)
+            copy.d = angular.toJson(copy.d)
+
+            parsed = dataUtilsService.parse(test)
+
+            expect(parsed).toEqual(test)
+
+    describe 'numberOrString(string)', ->
+
+        it 'should convert a string to a number if possible', ->
+            result = dataUtilsService.numberOrString('12')
+            expect(result).toBe(12)
+
+        it 'should return the string if it is not a number', ->
+            result = dataUtilsService.numberOrString('w3as')
+            expect(result).toBe('w3as')