Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

first commit

  • Loading branch information...
commit 42be953cf10b2074886778c4560ded1f14ecea74 0 parents
jeads authored April 26, 2012

Showing 153 changed files with 41,108 additions and 0 deletions. Show diff stats Hide diff stats

  1. 5  LICENSE.txt
  2. 113  README.md
  3. 0  __init__.py
  4. 0  controller/__init__.py
  5. 0  controller/admin/__init__.py
  6. 90  controller/admin/populate_summary_cache.py
  7. 44  controller/admin/populate_test_collections.py
  8. 510  model/DatazillaModel.py
  9. 81  model/Model.py
  10. 0  model/__init__.py
  11. 459  model/sql/graphs.json
  12. 57  model/t/DatazillaModelTest.py
  13. 0  model/t/__init__.py
  14. 0  webapp/__init__.py
  15. 0  webapp/apps/__init__.py
  16. 0  webapp/apps/datazilla/__init__.py
  17. 0  webapp/apps/datazilla/filters/__init__.py
  18. 0  webapp/apps/datazilla/filters/templatetags/__init__.py
  19. 60  webapp/apps/datazilla/filters/templatetags/dv_unorderedlist.py
  20. 0  webapp/apps/datazilla/management/__init__.py
  21. 0  webapp/apps/datazilla/management/commands/__init__.py
  22. 83  webapp/apps/datazilla/management/commands/build_nav.py
  23. 3  webapp/apps/datazilla/models.py
  24. 16  webapp/apps/datazilla/tests.py
  25. 9  webapp/apps/datazilla/urls.py
  26. 310  webapp/apps/datazilla/views.py
  27. 15  webapp/conf/bin/datazilla
  28. 6  webapp/conf/bin/start_datazilla
  29. 19  webapp/conf/etc/nginx/conf.d/datazilla.conf
  30. 24  webapp/conf/etc/nginx/fastcgi_params
  31. 101  webapp/conf/etc/rc.d/init.d/datazilla
  32. 13  webapp/conf/etc/sysconfig/datazilla
  33. 14  webapp/manage.py
  34. BIN  webapp/media/css/Aristo/images/bg_fallback.png
  35. BIN  webapp/media/css/Aristo/images/icon_sprite.png
  36. BIN  webapp/media/css/Aristo/images/progress_bar.gif
  37. BIN  webapp/media/css/Aristo/images/slider_handles.png
  38. BIN  webapp/media/css/Aristo/images/ui-icons_222222_256x240.png
  39. BIN  webapp/media/css/Aristo/images/ui-icons_454545_256x240.png
  40. 756  webapp/media/css/Aristo/jquery-ui-1.8.7.custom.css
  41. 108  webapp/media/css/ColVisAlt.css
  42. 216  webapp/media/css/common.css
  43. 312  webapp/media/css/datatables.css
  44. 397  webapp/media/css/dataviews.css
  45. 114  webapp/media/css/fg.menu.css
  46. 53  webapp/media/html/control_panels/named_fields.html
  47. 81  webapp/media/html/control_panels/test_selector.html
  48. 28  webapp/media/html/nav_menu.html
  49. BIN  webapp/media/images/back_disabled.jpg
  50. BIN  webapp/media/images/back_enabled.jpg
  51. BIN  webapp/media/images/background.png
  52. BIN  webapp/media/images/bhview_panel.png
  53. BIN  webapp/media/images/col_button.png
  54. BIN  webapp/media/images/dino3.png
  55. BIN  webapp/media/images/forward_disabled.jpg
  56. BIN  webapp/media/images/forward_enabled.jpg
  57. BIN  webapp/media/images/sort_asc.png
  58. BIN  webapp/media/images/sort_asc_disabled.png
  59. BIN  webapp/media/images/sort_both.png
  60. BIN  webapp/media/images/sort_desc.png
  61. BIN  webapp/media/images/sort_desc_disabled.png
  62. BIN  webapp/media/images/spinner.gif
  63. BIN  webapp/media/images/title.png
  64. 105  webapp/media/js/data_views/Bases.js
  65. 176  webapp/media/js/data_views/ConnectionsComponent.js
  66. 828  webapp/media/js/data_views/DataAdapterCollection.js
  67. 493  webapp/media/js/data_views/DataViewCollection.js
  68. 1,834  webapp/media/js/data_views/DataViewComponent.js
  69. 129  webapp/media/js/data_views/DataViewPage.js
  70. 28  webapp/media/js/data_views/HelpPage.js
  71. 1,657  webapp/media/js/data_views/VisualizationCollection.js
  72. 386  webapp/media/js/plugins/box.js
  73. 9,342  webapp/media/js/plugins/d3.v2.js
  74. 4  webapp/media/js/plugins/d3.v2.min.js
  75. 1,020  webapp/media/js/plugins/datatables/ColReorderWithResize.js
  76. 875  webapp/media/js/plugins/datatables/ColVis.js
  77. 7,465  webapp/media/js/plugins/datatables/jquery.dataTables.js
  78. 151  webapp/media/js/plugins/datatables/jquery.dataTables.min.js
  79. 639  webapp/media/js/plugins/fg.menu.js
  80. 1,201  webapp/media/js/plugins/flot-0.7/API.txt
  81. 76  webapp/media/js/plugins/flot-0.7/FAQ.txt
  82. 22  webapp/media/js/plugins/flot-0.7/LICENSE.txt
  83. 9  webapp/media/js/plugins/flot-0.7/Makefile
  84. 508  webapp/media/js/plugins/flot-0.7/NEWS.txt
  85. 137  webapp/media/js/plugins/flot-0.7/PLUGINS.txt
  86. 90  webapp/media/js/plugins/flot-0.7/README.txt
  87. 143  webapp/media/js/plugins/flot-0.7/examples/ajax.html
  88. 75  webapp/media/js/plugins/flot-0.7/examples/annotating.html
  89. BIN  webapp/media/js/plugins/flot-0.7/examples/arrow-down.gif
  90. BIN  webapp/media/js/plugins/flot-0.7/examples/arrow-left.gif
  91. BIN  webapp/media/js/plugins/flot-0.7/examples/arrow-right.gif
  92. BIN  webapp/media/js/plugins/flot-0.7/examples/arrow-up.gif
  93. 38  webapp/media/js/plugins/flot-0.7/examples/basic.html
  94. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-1.json
  95. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-2.json
  96. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-3.json
  97. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-4.json
  98. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-5.json
  99. 4  webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth.json
  100. 4  webapp/media/js/plugins/flot-0.7/examples/data-japan-gdp-growth.json
  101. 4  webapp/media/js/plugins/flot-0.7/examples/data-usa-gdp-growth.json
  102. 75  webapp/media/js/plugins/flot-0.7/examples/graph-types.html
  103. BIN  webapp/media/js/plugins/flot-0.7/examples/hs-2004-27-a-large_web.jpg
  104. 45  webapp/media/js/plugins/flot-0.7/examples/image.html
  105. 44  webapp/media/js/plugins/flot-0.7/examples/index.html
  106. 97  webapp/media/js/plugins/flot-0.7/examples/interacting-axes.html
  107. 93  webapp/media/js/plugins/flot-0.7/examples/interacting.html
  108. 6  webapp/media/js/plugins/flot-0.7/examples/layout.css
  109. 60  webapp/media/js/plugins/flot-0.7/examples/multiple-axes.html
  110. 118  webapp/media/js/plugins/flot-0.7/examples/navigate.html
  111. 57  webapp/media/js/plugins/flot-0.7/examples/percentiles.html
  112. 756  webapp/media/js/plugins/flot-0.7/examples/pie.html
  113. 83  webapp/media/js/plugins/flot-0.7/examples/realtime.html
  114. 61  webapp/media/js/plugins/flot-0.7/examples/resize.html
  115. 114  webapp/media/js/plugins/flot-0.7/examples/selection.html
  116. 61  webapp/media/js/plugins/flot-0.7/examples/setting-options.html
  117. 77  webapp/media/js/plugins/flot-0.7/examples/stacking.html
  118. 49  webapp/media/js/plugins/flot-0.7/examples/symbols.html
  119. 54  webapp/media/js/plugins/flot-0.7/examples/thresholding.html
  120. 71  webapp/media/js/plugins/flot-0.7/examples/time.html
  121. 95  webapp/media/js/plugins/flot-0.7/examples/tracking.html
  122. 98  webapp/media/js/plugins/flot-0.7/examples/turning-series.html
  123. 90  webapp/media/js/plugins/flot-0.7/examples/visitors.html
  124. 98  webapp/media/js/plugins/flot-0.7/examples/zooming.html
  125. 1,427  webapp/media/js/plugins/flot-0.7/excanvas.js
  126. 1  webapp/media/js/plugins/flot-0.7/excanvas.min.js
  127. 179  webapp/media/js/plugins/flot-0.7/jquery.colorhelpers.js
  128. 1  webapp/media/js/plugins/flot-0.7/jquery.colorhelpers.min.js
  129. 401  webapp/media/js/plugins/flot-0.7/jquery.flot.axislabels.js
  130. 167  webapp/media/js/plugins/flot-0.7/jquery.flot.crosshair.js
  131. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.crosshair.min.js
  132. 183  webapp/media/js/plugins/flot-0.7/jquery.flot.fillbetween.js
  133. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.fillbetween.min.js
  134. 238  webapp/media/js/plugins/flot-0.7/jquery.flot.image.js
  135. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.image.min.js
  136. 2,599  webapp/media/js/plugins/flot-0.7/jquery.flot.js
  137. 6  webapp/media/js/plugins/flot-0.7/jquery.flot.min.js
  138. 336  webapp/media/js/plugins/flot-0.7/jquery.flot.navigate.js
  139. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.navigate.min.js
  140. 187  webapp/media/js/plugins/flot-0.7/jquery.flot.orderBars.js
  141. 750  webapp/media/js/plugins/flot-0.7/jquery.flot.pie.js
  142. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.pie.min.js
  143. 60  webapp/media/js/plugins/flot-0.7/jquery.flot.resize.js
  144. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.resize.min.js
  145. 344  webapp/media/js/plugins/flot-0.7/jquery.flot.selection.js
  146. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.selection.min.js
  147. 184  webapp/media/js/plugins/flot-0.7/jquery.flot.stack.js
  148. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.stack.min.js
  149. 70  webapp/media/js/plugins/flot-0.7/jquery.flot.symbol.js
  150. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.symbol.min.js
  151. 103  webapp/media/js/plugins/flot-0.7/jquery.flot.threshold.js
  152. 1  webapp/media/js/plugins/flot-0.7/jquery.flot.threshold.min.js
5  LICENSE.txt
... ...
@@ -0,0 +1,5 @@
  1
+#####
  2
+# This Source Code Form is subject to the terms of the Mozilla Public
  3
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4
+# You can obtain one at http://mozilla.org/MPL/2.0/.
  5
+#####
113  README.md
Source Rendered
... ...
@@ -0,0 +1,113 @@
  1
+#Datazilla
  2
+Datazilla is a generic system for managing and visualizing data.  The fundamental unit of data display in the user interface is called a data view.  Data views can display data in any number of ways: tabular or graphical.  Data views can also send signals to one another enabling the user to maintain visual context across multiple graphical displays of different data types.  Each data view shares a toolbar that abstracts navigation, data presentation controls, and visual presentation.  A prototype of datazilla was first developed in an application called [bughunter] [1].
  3
+
  4
+This project includes a model, webservice, and web based user interface, and eventually it will support a local development environment. 
  5
+
  6
+This is a work in progress and will likely see a number of structural changes.  It is currently being developed to manage [Talos] [2] test data, a performance testing framework developed by mozilla for testing software products.
  7
+
  8
+[1]: https://wiki.mozilla.org/Auto-tools/Projects/BugHunter  "bughunter"
  9
+[2]: https://wiki.mozilla.org/Buildbot/Talos "Talos"
  10
+
  11
+##Architecture
  12
+At a top level datazilla can be described with three different parts: model, webservice, and UI.
  13
+
  14
+All environment information is stored in datazilla/webapp/conf/etc/sysconfig/datazilla.  Appropriate system information for the system should be added to this file and then it copy to /etc/sysconfig/datazilla.  It needs to be source'd before running any component of the system.  Most of the environment variables are self explanatory. 
  15
+
  16
+The environment variable called DATAZILLA_DEBUG, when set to true, causes all scripts and webservice methods to write out the full SQL, execution time, and host name that are being executed at runtime.  This is handy for debugging any component in the system.
  17
+
  18
+###Model
  19
+The datazilla model classes rely on a module called [datasource] [3].  This module encapsulates SQL manipulation.  All of the SQL used by the system is stored in a JSON file format in found in /datazilla/model/sql.  There can be any number of SQL files stored in this format.  The JSON structure allows a host type to be associated with each SQL statement.
  20
+
  21
+[3]: https://github.com/jeads/datasource "datasource"
  22
+
  23
+###UI
  24
+
  25
+####Building the Navigation Menu And Defining Data Views
  26
+New data views and collections of dataviews can be defined in The navigation menu 
  27
+
  28
+####Building the Cached Summaries
  29
+
  30
+
  31
+####JS
  32
+
  33
+#####Class Structures
  34
+The javascript that implements the user interface is constructed using a page/component/collection pattern thingy... whatever that means.  This pattern was found to be very useful in separating out the required functionality, below is a brief definition of what that means in the data view UI architecture.
  35
+
  36
+######Page
  37
+Manages the DOM ready event, implements any top level initialization that's required for the page.  An instance of the page class is the only global variable that other components can access, if they're playing nice.  The page class instance is responsible for instantiating components and storing them in attributes.  The page class also holds any data structures that need to be globally accessible to component classes. 
  38
+
  39
+######Component
  40
+Contains the public interface of the component.  A component can encapsulate any functional subset/unit provided in a page.  The component will typically have an instance of a View and Model class.  The component class is also responsible for any required event binding.
  41
+
  42
+######View
  43
+A component's view class manages interfacing with the DOM. Any CSS class names or HTML id's are defined as attributes of the view.  Any HTML element modification is controlled with this class.
  44
+
  45
+######Model
  46
+A component's model manages any asynchronous data retrieval and large data structure manipulation.
  47
+
  48
+######Collection
  49
+A class for managing a collection of Components or classes of any type.  A collection can also have a model/view if appropriate.
  50
+
  51
+######Client Application (datazilla/webapp/media/js/data_views)
  52
+This is not a complete file or class listing but is intended to give a top level description of the design pattern thingy of the data view javascript and what the basic functional responsibility of the pages/components/collections are.
  53
+
  54
+#######DataViewPage.js 
  55
+DataViewPage Class - Manages the DOM ready event, component initialization, and retrieval of the views.json structure that is used by different components.
  56
+
  57
+#######Bases.js
  58
+Design Pattern Base Classes - Contains the base classes for Page, Component, Model, View etc...
  59
+                                                                  
  60
+#######DataViewComponent.js 
  61
+DataViewComponent Class - Encapsulates the behavior of a single data view using a model/view and provides a public interface for data view functionality.  Manages event binding and registration.
  62
+
  63
+DVViewView Class - Encapsulates all DOM interaction required by a data view.
  64
+
  65
+BHViewModel Class - Encapsulates asynchronous server communication and data structure manipulation/retrieval.
  66
+
  67
+#######DataViewCollection.js 
  68
+
  69
+DataViewCollection Class - Manages operations on a collection of data views using a model/view including instantiating view collections.  
  70
+
  71
+DataViewCollectionView Class - Encapsulates all DOM interaction required by the collection.
  72
+
  73
+DataViewCollectionModel Class - Provides an interface to the datastructures holding all data views and their associated parent/child relationships.
  74
+
  75
+#######DataAdapterCollection.js
  76
+
  77
+DataAdapterCollection Class - Collection of DataViewAdapter class instances. 
  78
+
  79
+BHViewAdapter Class - Base class for all BHViewAdapters.  Manages shared view idiosyncratic behavior like what fields go in the control panel and how to populate/retrieve them for signaling behavior.
  80
+
  81
+CrashesAdapter Class - Derived class of BHViewAdapter.  Encapsulates unique behavior for crash data views.
  82
+
  83
+UrlAdapter Class - Derived class of BHViewAdapter. Encapsulates unique behavior for views containing URL summaries.
  84
+
  85
+##Installation
  86
+1. Add system info to appropriate files in datazilla/webapp/conf/etc.  Copy the files to there appropriate location under /etc.
  87
+
  88
+2. Start the datazilla, nginx, and memcached services.  There is a startup script for the datazilla and nginx services in datazilla/webapp/conf/bin.
  89
+
  90
+##RHEL6 Configuration
  91
+
  92
+This configuration was done on a RHEL6 VM.
  93
+
  94
+1. cat /etc/redhat-release to get the correct version of EPEL
  95
+
  96
+2. rpm -Uvh http://download.fedora.redhat.com/pub/epel/6/i386/epel-release-6-5.noarch.rpm
  97
+
  98
+3. yum install nginx fcgi Django Django-doc python-docutils MySQL-python python-flup
  99
+
  100
+4. yum install python-setuptools spawn-fcgi
  101
+
  102
+5. yum install mysql.x86_64 mysql-server.x86_64 mysql-devel.x86_64
  103
+
  104
+6. yum install git
  105
+
  106
+7. git clone https://github.com/jeads/datasource, python setup.py install
  107
+
  108
+8. yum install memcached.x86_64 python-memcached.noarch
  109
+
  110
+9. Modify the contents of files in the datazilla/webapp/conf/etc directory to meet the needs of
  111
+   your system and then copy the files to their corresponding locations under /etc.
  112
+
  113
+
0  __init__.py
No changes.
0  controller/__init__.py
No changes.
0  controller/admin/__init__.py
No changes.
90  controller/admin/populate_summary_cache.py
... ...
@@ -0,0 +1,90 @@
  1
+import os
  2
+import sys
  3
+import time
  4
+import datetime
  5
+import json
  6
+import memcache
  7
+import zlib
  8
+
  9
+from optparse import OptionParser
  10
+from datazilla.model.DatazillaModel import DatazillaModel
  11
+
  12
+"""
  13
+This script builds the test run summary data structure for
  14
+a 7 and 30 day period interval for every product/branch/version.
  15
+
  16
+These data structures are stored in the summary_cache table.  They
  17
+need to persist if the memcache goes down, they take several minutes
  18
+to generate.  As the quantity of data grows this will likely take
  19
+significantly longer.
  20
+"""
  21
+
  22
+def cacheTestSummaries():
  23
+
  24
+   gm = DatazillaModel('graphs.json')
  25
+   dataIter = gm.getAllSummaryCacheData()
  26
+
  27
+   mc = memcache.Client([os.environ["DATAZILLA_MEMCACHED"]], debug=0)
  28
+
  29
+   for d in dataIter:
  30
+      for data in d:
  31
+         key = DatazillaModel.getCacheKey( data['item_id'], data['item_data'] )
  32
+         rv = mc.set(key, zlib.compress( data['value'] ))
  33
+         if not rv:
  34
+            sys.stderr.write("ERROR: Failed to store object in memcache: %s, %s\n" % ( str(data['item_id']), data['item_data'] ) ) 
  35
+
  36
+   gm.disconnect()
  37
+
  38
+def buildTestSummaries():
  39
+
  40
+   gm = DatazillaModel('graphs.json')
  41
+
  42
+   timeRanges = DatazillaModel.getTimeRanges()
  43
+
  44
+   products = gm.getProducts()
  45
+
  46
+   for productName in products:
  47
+
  48
+      for tr in ['days_7', 'days_30']:
  49
+
  50
+         table = gm.getTestRunSummary(str( timeRanges[tr]['start']),
  51
+                                      str( timeRanges[tr]['stop']),
  52
+                                      [ products[ productName ] ],
  53
+                                      [],
  54
+                                      [])
  55
+
  56
+         jsonData = json.dumps( table )
  57
+
  58
+         gm.setSummaryCache( products[ productName ], tr, jsonData )
  59
+
  60
+   gm.disconnect()
  61
+
  62
+if __name__ == '__main__':
  63
+
  64
+   usage = """usage: %prog [options] --build --cache --verbose"""
  65
+   parser = OptionParser(usage=usage)
  66
+
  67
+   parser.add_option('-b', 
  68
+                     '--build', 
  69
+                     action='store_true', 
  70
+                     dest='build',
  71
+                     default=False, 
  72
+                     type=None,
  73
+                     help="Build the test run summaries and store them in the database.")
  74
+
  75
+   parser.add_option('-c', 
  76
+                     '--cache', 
  77
+                     action='store_true', 
  78
+                     dest='cache',
  79
+                     default=False, 
  80
+                     type=None,
  81
+                     help="Update the test run summaries in memcached")
  82
+
  83
+   (options, args) = parser.parse_args()
  84
+
  85
+   if options.build:
  86
+      buildTestSummaries()
  87
+
  88
+   if options.cache:
  89
+      cacheTestSummaries()
  90
+
44  controller/admin/populate_test_collections.py
... ...
@@ -0,0 +1,44 @@
  1
+import os
  2
+import sys
  3
+
  4
+from optparse import OptionParser
  5
+from datazilla.model.DatazillaModel import DatazillaModel
  6
+
  7
+def loadTestCollection():
  8
+
  9
+   gm = DatazillaModel('graphs.json')
  10
+
  11
+   products = gm.getProducts('id')
  12
+
  13
+   for productName in products:
  14
+
  15
+      if products[ productName ]['product'] and \
  16
+         products[ productName ]['version'] and \
  17
+         products[ productName ]['branch']:
  18
+
  19
+         name = "%s %s %s" % (products[ productName ]['product'],
  20
+                              products[ productName ]['version'],
  21
+                              products[ productName ]['branch'])
  22
+
  23
+         id = gm.setData('set_test_collection', [ name, "", name ])
  24
+         gm.setData('set_test_collection_map', [ id, products[ productName ]['id'] ])
  25
+
  26
+   gm.disconnect()
  27
+
  28
+if __name__ == '__main__':
  29
+
  30
+   usage = """usage: %prog [options] --load --verbose"""
  31
+   parser = OptionParser(usage=usage)
  32
+
  33
+   parser.add_option('-l', 
  34
+                     '--load', 
  35
+                     action='store_true', 
  36
+                     dest='load',
  37
+                     default=False, 
  38
+                     type=None,
  39
+                     help="Identitfy new product branches and add them as test collections.")
  40
+
  41
+   (options, args) = parser.parse_args()
  42
+
  43
+   if options.load:
  44
+      loadTestCollection()
510  model/DatazillaModel.py
... ...
@@ -0,0 +1,510 @@
  1
+#####
  2
+# This Source Code Form is subject to the terms of the Mozilla Public
  3
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4
+# You can obtain one at http://mozilla.org/MPL/2.0/.
  5
+#####
  6
+import datetime
  7
+import time
  8
+
  9
+from datazilla.model.Model import Model
  10
+
  11
+class DatazillaModel(Model):
  12
+
  13
+   @staticmethod
  14
+   def getIdString(idList):
  15
+      ####
  16
+      # idList = [1,2,3,4,5]
  17
+      #
  18
+      # Cast each element in the list to a string.
  19
+      #
  20
+      # Join the strings on a ',' and return '1,2,3,4,5'
  21
+      #####
  22
+      return ','.join( map( lambda i: str(i), idList ) )
  23
+
  24
+   @staticmethod
  25
+   def getIdList(idString):
  26
+      ####
  27
+      # idString = '1,2,3,4,5'
  28
+      #
  29
+      # Split the string on a ',' to get a list.
  30
+      #
  31
+      # Cast each element in the list to an int.
  32
+      #
  33
+      # If an element generates a ValueError on the cast return an empty list.
  34
+      ####
  35
+      idList = []
  36
+
  37
+      try:
  38
+         idList = map( lambda s: int(s), idString.split(',') )
  39
+      except ValueError:
  40
+         ##Cast to int failed, must not be a number##
  41
+         pass
  42
+
  43
+      return idList
  44
+
  45
+   @staticmethod
  46
+   def getCacheKey(itemId, itemData):
  47
+      return str(itemId) + '_' + str(itemData)
  48
+
  49
+   @staticmethod
  50
+   def getTimeRanges():
  51
+
  52
+      #############
  53
+      #  time.time() is used to generate the unix timestamp 
  54
+      #  associated with the json structure pushed to the database
  55
+      #  by a talos bot.  So all time ranges need to be computed in
  56
+      #  seconds since the epoch
  57
+      ############
  58
+
  59
+      now = int( time.time() )
  60
+
  61
+      timeRanges = { 'days_7': { 'start':now, 'stop':now - 604800  },      
  62
+                     'days_30': { 'start':now, 'stop':now - 2592000 },    
  63
+                     'days_60': { 'start':now, 'stop':now - 5184000 }, 
  64
+                     'days_90': { 'start':now, 'stop':now - 7776000 },  
  65
+                     'days_360': { 'start':now, 'stop':now - 31557600 } }
  66
+
  67
+      return timeRanges
  68
+
  69
+   def __init__(self, sqlFileName):
  70
+
  71
+      Model.__init__(self, sqlFileName)
  72
+
  73
+   def getProductTestOsMap(self):
  74
+
  75
+      productTuple = self.dhub.execute(proc='graphs.selects.get_product_test_os_map',
  76
+                                       debug_show=self.DEBUG,
  77
+                                       return_type='tuple') 
  78
+
  79
+      return productTuple
  80
+
  81
+   def getOperatingSystems(self, keyColumn=None):
  82
+
  83
+      operatingSystems = dict()
  84
+      if keyColumn:
  85
+         operatingSystems = self.dhub.execute(proc='graphs.selects.get_operating_systems',
  86
+                                     debug_show=self.DEBUG,
  87
+                                     key_column=keyColumn,
  88
+                                     return_type='dict') 
  89
+      else:
  90
+         osTuple = self.dhub.execute(proc='graphs.selects.get_operating_systems',
  91
+                                     debug_show=self.DEBUG,
  92
+                                     return_type='tuple') 
  93
+
  94
+         operatingSystems = self._getUniqueKeyDict(osTuple, ['name', 'version'])
  95
+
  96
+      return operatingSystems
  97
+
  98
+   def getTests(self, keyColumn='name'):
  99
+
  100
+      testDict = self.dhub.execute(proc='graphs.selects.get_tests',
  101
+                                  debug_show=self.DEBUG,
  102
+                                  key_column=keyColumn,
  103
+                                  return_type='dict') 
  104
+
  105
+      return testDict
  106
+
  107
+   def getProducts(self, keyColumn=None):
  108
+
  109
+      products = dict()
  110
+      if keyColumn:
  111
+         products = self.dhub.execute(proc='graphs.selects.get_product_data',
  112
+                                           debug_show=self.DEBUG,
  113
+                                           key_column=keyColumn,
  114
+                                           return_type='dict') 
  115
+      else:
  116
+         productsTuple = self.dhub.execute(proc='graphs.selects.get_product_data',
  117
+                                           debug_show=self.DEBUG,
  118
+                                           return_type='tuple') 
  119
+
  120
+         products = self._getUniqueKeyDict(productsTuple, ['product', 'branch', 'version'])
  121
+
  122
+      return products 
  123
+
  124
+   def getMachines(self):
  125
+      machinesDict = self.dhub.execute(proc='graphs.selects.get_machines',
  126
+                                    debug_show=self.DEBUG,
  127
+                                    key_column='name',
  128
+                                    return_type='dict') 
  129
+
  130
+      return machinesDict
  131
+      
  132
+   def getOptions(self):
  133
+      optionsDict = self.dhub.execute(proc='graphs.selects.get_options',
  134
+                                    debug_show=self.DEBUG,
  135
+                                    key_column='name',
  136
+                                    return_type='dict') 
  137
+
  138
+      return optionsDict
  139
+
  140
+   def getPages(self):
  141
+      pagesDict = self.dhub.execute(proc='graphs.selects.get_pages',
  142
+                                    debug_show=self.DEBUG,
  143
+                                    key_column='url',
  144
+                                    return_type='dict') 
  145
+
  146
+      return pagesDict
  147
+
  148
+   def getAuxData(self):
  149
+      auxDataDict = self.dhub.execute(proc='graphs.selects.get_aux_data',
  150
+                                    debug_show=self.DEBUG,
  151
+                                    key_column='name',
  152
+                                    return_type='dict') 
  153
+
  154
+      return auxDataDict
  155
+
  156
+   def getReferenceData(self):
  157
+
  158
+      referenceData = dict( operating_systems=self.getOperatingSystems(),
  159
+                            tests=self.getTests(),
  160
+                            products=self.getProducts(),
  161
+                            machines=self.getMachines(),
  162
+                            options=self.getOptions(),
  163
+                            pages=self.getPages(),
  164
+                            aux_data=self.getAuxData())
  165
+
  166
+      return referenceData
  167
+
  168
+   def getTestCollections(self):
  169
+
  170
+      testCollectionTuple = self.dhub.execute(proc='graphs.selects.get_test_collections',
  171
+                                              debug_show=self.DEBUG,
  172
+                                              return_type='tuple') 
  173
+
  174
+      testCollection = dict()
  175
+      for data in testCollectionTuple:
  176
+
  177
+         if data['id'] not in testCollection:
  178
+
  179
+            testCollection[ data['id'] ] = dict()
  180
+            testCollection[ data['id'] ]['name'] = data['name']
  181
+            testCollection[ data['id'] ]['description'] = data['description']
  182
+            testCollection[ data['id'] ]['data'] = []
  183
+
  184
+         testCollection[ data['id'] ]['data'].append( { 'test_id':data['test_id'],
  185
+                                                        'name':data['name'],
  186
+                                                        'product_id':data['product_id'],
  187
+                                                        'operating_system_id':data['operating_system_id'] } )
  188
+
  189
+
  190
+      return testCollection
  191
+
  192
+   def getTestReferenceData(self):
  193
+
  194
+      referenceData = dict( operating_systems=self.getOperatingSystems('id'),
  195
+                            tests=self.getTests('id'),
  196
+                            products=self.getProducts('id'),
  197
+                            product_test_os_map=self.getProductTestOsMap(),
  198
+                            test_collections=self.getTestCollections())
  199
+
  200
+      return referenceData
  201
+
  202
+   def getTestRunSummary(self, start, end, productIds, operatingSystemIds, testIds):
  203
+
  204
+      colData = { 'b.product_id':DatazillaModel.getIdString(productIds),
  205
+                  'b.operating_system_id':DatazillaModel.getIdString(operatingSystemIds),
  206
+                  'tr.test_id':DatazillaModel.getIdString(testIds) }
  207
+
  208
+      rep = self.buildReplacement(colData)
  209
+
  210
+      testRunSummaryTable = self.dhub.execute(proc='graphs.selects.get_test_run_summary',
  211
+                                              debug_show=self.DEBUG,
  212
+                                              replace=[ str(end), str(start), rep ],
  213
+                                              return_type='table') 
  214
+
  215
+      return testRunSummaryTable
  216
+
  217
+   def getAllTestRuns(self):
  218
+
  219
+      testRunSummaryTable = self.dhub.execute(proc='graphs.selects.get_all_test_runs',
  220
+                                              debug_show=self.DEBUG,
  221
+                                              return_type='table') 
  222
+
  223
+      return testRunSummaryTable
  224
+
  225
+   def getTestRunValues(self, testRunId):
  226
+
  227
+      testRunValueTable = self.dhub.execute(proc='graphs.selects.get_test_run_values',
  228
+                                            debug_show=self.DEBUG,
  229
+                                            placeholders=[ testRunId ],
  230
+                                            return_type='table') 
  231
+
  232
+      return testRunValueTable
  233
+
  234
+   def getTestRunValueSummary(self, testRunId):
  235
+
  236
+      testRunValueTable = self.dhub.execute(proc='graphs.selects.get_test_run_value_summary',
  237
+                                            debug_show=self.DEBUG,
  238
+                                            placeholders=[ testRunId ],
  239
+                                            return_type='table') 
  240
+
  241
+      return testRunValueTable
  242
+
  243
+   def getPageValues(self, testRunId, pageId):
  244
+
  245
+      pageValuesTable = self.dhub.execute(proc='graphs.selects.get_page_values',
  246
+                                          debug_show=self.DEBUG,
  247
+                                          placeholders=[ testRunId, pageId ],
  248
+                                          return_type='table') 
  249
+
  250
+      return pageValuesTable
  251
+      
  252
+   def getSummaryCacheData(self, itemId, itemData):
  253
+
  254
+      cachedData = self.dhub.execute(proc='graphs.selects.get_summary_cache',
  255
+                                     debug_show=self.DEBUG,
  256
+                                     replace=[ itemId, itemData ],
  257
+                                     return_type='tuple') 
  258
+
  259
+      return cachedData
  260
+
  261
+   def getAllSummaryCacheData(self):
  262
+
  263
+      dataIter = self.dhub.execute(proc='graphs.selects.get_all_summary_cache_data',
  264
+                                   debug_show=self.DEBUG,
  265
+                                   chunk_size=5,
  266
+                                   chunk_source="summary_cache.id",
  267
+                                   return_type='tuple')
  268
+
  269
+
  270
+      return dataIter
  271
+
  272
+   def setTestCollection(self, name, description):
  273
+
  274
+
  275
+      return id
  276
+
  277
+   def setSummaryCache(self, itemId, itemData, value):
  278
+
  279
+      nowDatetime = str( datetime.datetime.now() )
  280
+
  281
+      self.dhub.execute(proc='graphs.inserts.set_summary_cache',
  282
+                        debug_show=self.DEBUG,
  283
+                        placeholders=[ itemId, 
  284
+                                       itemData, 
  285
+                                       value, 
  286
+                                       nowDatetime, 
  287
+                                       value, 
  288
+                                       nowDatetime ]) 
  289
+
  290
+   def loadTestData(self, data, jsonData):
  291
+
  292
+      ##Get the reference data##
  293
+      refData = self.getReferenceData()
  294
+
  295
+      ##Get/Set reference info##
  296
+      refData['test_id'] = self._getTestId(data, refData)
  297
+      refData['option_id_map'] = self._getOptionIds(data, refData)
  298
+      refData['operating_system_id'] = self._getOsId(data, refData)
  299
+      refData['product_id'] = self._getProductId(data, refData)
  300
+      refData['machine_id'] = self._getMachineId(data, refData)
  301
+
  302
+      refData['build_id'] = self._setBuildData(data, refData)
  303
+      refData['test_run_id'] = self._setTestRunData(data, refData)
  304
+
  305
+      self._setOptionData(data, refData)
  306
+      self._setTestValues(data, refData)
  307
+      self._setTestAuxData(data, refData)
  308
+      self._setTestData(jsonData, refData)
  309
+
  310
+   def _setTestData(self, jsonData, refData):
  311
+   
  312
+      self.setData('set_test_data', [refData['test_run_id'], jsonData])
  313
+      
  314
+   def _setTestAuxData(self, data, refData):
  315
+
  316
+      if 'results_aux' in data:
  317
+
  318
+         for auxData in data['results_aux']:
  319
+            auxDataId = self._getAuxId(auxData, refData)
  320
+            auxValues = data['results_aux'][auxData]
  321
+
  322
+            for index in range(0, len(auxValues)):
  323
+
  324
+               stringData = ""
  325
+               numericData = 0
  326
+               if self.isNumber(auxValues[index]):
  327
+                  numericData = auxValues[index]
  328
+               else:
  329
+                  stringData = auxValues[index]
  330
+
  331
+               self.setData('set_aux_values', [refData['test_run_id'],
  332
+                                                index + 1,
  333
+                                                auxDataId,
  334
+                                                numericData,
  335
+                                                stringData])
  336
+
  337
+   def _setTestValues(self, data, refData):
  338
+
  339
+      for page in data['results']:
  340
+
  341
+         pageId = self._getPageId(page, refData)
  342
+         
  343
+         values = data['results'][page]
  344
+
  345
+         for index in range(0, len(values)):
  346
+
  347
+            value = values[index]
  348
+            self.setData('set_test_values', [refData['test_run_id'],
  349
+                                              index + 1,
  350
+                                              pageId, 
  351
+                                              ##Need to get the value id into the json##
  352
+                                              1,
  353
+                                              value])
  354
+
  355
+   def _getAuxId(self, auxData, refData):
  356
+      
  357
+      auxId = 0
  358
+      try:
  359
+         if auxData in refData['aux_data']:
  360
+            auxId = refData['aux_data'][auxData]['id']
  361
+         else:
  362
+            auxId = self.setData('set_aux_data', [refData['test_id'], auxData])
  363
+            
  364
+      except KeyError:
  365
+         raise
  366
+      else:
  367
+         return auxId
  368
+      
  369
+   def _getPageId(self, page, refData):
  370
+      
  371
+      pageId = 0
  372
+      try:
  373
+         if page in refData['pages']:
  374
+            pageId = refData['pages'][page]['id']
  375
+         else:
  376
+            pageId = self.setData('set_pages_data', [refData['test_id'], page])
  377
+
  378
+      except KeyError:
  379
+         raise
  380
+      else:
  381
+         return pageId
  382
+
  383
+   def _setOptionData(self, data, refData):
  384
+
  385
+      for option in data['testrun']['options']:
  386
+         id = refData['option_id_map'][option]['id']
  387
+         value = data['testrun']['options'][option]
  388
+         self.setData('set_test_option_values', [refData['test_run_id'],
  389
+                                                  id,
  390
+                                                  value])
  391
+      
  392
+   def _setBuildData(self, data, refData):
  393
+      
  394
+      buildId = self.setData('set_build_data', [ refData['operating_system_id'],
  395
+                                                  refData['product_id'], 
  396
+                                                  refData['machine_id'],
  397
+                                                  data['test_build']['id'],
  398
+                                                  data['test_machine']['platform'],
  399
+                                                  data['test_build']['revision'],
  400
+                                                  ##Need to get the build_type into the json##
  401
+                                                  'debug',
  402
+                                                  ##Need to get the build_date into the json##
  403
+                                                  int(time()) ] )
  404
+      
  405
+      return buildId
  406
+      
  407
+   def _setTestRunData(self, data, refData):
  408
+
  409
+      testRunId = self.setData('set_test_run_data', [ refData['test_id'],
  410
+                                                       refData['build_id'],
  411
+                                                       data['test_build']['revision'],
  412
+                                                       data['testrun']['date'] ])
  413
+
  414
+      return testRunId
  415
+
  416
+   def _getMachineId(self, data, refData):
  417
+
  418
+      machineId = 0
  419
+      try:
  420
+         name = data['test_machine']['name']
  421
+         if name in refData['machines']:
  422
+            machineId = refData['machines'][ name ]['id']
  423
+         else:
  424
+            machineId = self.setData('set_machine_data', [ name, int(time()) ])
  425
+
  426
+      except KeyError:
  427
+         raise
  428
+
  429
+      else:
  430
+         return machineId
  431
+
  432
+   def _getTestId(self, data, refData):
  433
+      testId = 1 
  434
+      try:
  435
+         if data['testrun']['suite'] in refData['tests']:
  436
+            testId = refData['tests'][ data['testrun']['suite'] ]['id']
  437
+         else:
  438
+            version = 1 
  439
+            if 'suite_version' in data['testrun']:
  440
+               version = int(data['testrun']['suite_version'])
  441
+
  442
+            testId = self.setData('set_test', [ data['testrun']['suite'], version ])
  443
+      except KeyError:
  444
+         raise 
  445
+      else:
  446
+         return testId
  447
+
  448
+   def _getOsId(self, data, refData):
  449
+
  450
+      osId = 0
  451
+      try:
  452
+         osName = data['test_machine']['os']
  453
+         osVersion = data['test_machine']['osversion']
  454
+         osKey = osName + osVersion
  455
+         if osKey in refData['operating_systems']:
  456
+            osId = refData['operating_systems'][osKey]
  457
+         else:
  458
+            osId = self.setData('set_operating_system', [ osName, osVersion ])
  459
+
  460
+      except KeyError:
  461
+         raise
  462
+
  463
+      else:
  464
+         return osId
  465
+
  466
+   def _getOptionIds(self, data, refData):
  467
+      optionIds = dict()
  468
+      try:
  469
+         for option in data['testrun']['options']:
  470
+            if option in refData['options']:
  471
+               optionIds[ option ] = refData['options'][option]
  472
+            else:
  473
+               testId = self.setData('set_option', [ option ])
  474
+               optionIds[ option ] = testId 
  475
+      except KeyError:
  476
+         raise
  477
+      else:
  478
+         return optionIds
  479
+
  480
+   def _getProductId(self, data, refData):
  481
+
  482
+      productId = 0
  483
+
  484
+      try:
  485
+         product = data['test_build']['name']
  486
+         branch = data['test_build']['branch']
  487
+         version = data['test_build']['version']
  488
+
  489
+         productKey = product + branch + version
  490
+
  491
+         if productKey in refData['products']:
  492
+            productId = refData['products'][productKey]
  493
+         else:
  494
+            productId = self.setData('set_product_data', [ product, branch, version ])
  495
+
  496
+      except KeyError:
  497
+         raise
  498
+      else:
  499
+         return productId
  500
+
  501
+   def _getUniqueKeyDict(self, dataTuple, keyStrings):   
  502
+
  503
+      dataDict = dict()
  504
+      for data in dataTuple:
  505
+         uniqueKey = ""
  506
+         for key in keyStrings:
  507
+            uniqueKey += str(data[key])
  508
+         dataDict[ uniqueKey ] = data['id']
  509
+      return dataDict
  510
+
81  model/Model.py
... ...
@@ -0,0 +1,81 @@
  1
+#####
  2
+# This Source Code Form is subject to the terms of the Mozilla Public
  3
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4
+# You can obtain one at http://mozilla.org/MPL/2.0/.
  5
+#####
  6
+import sys
  7
+import os
  8
+import re
  9
+
  10
+from datasource.bases.BaseHub import BaseHub
  11
+from datasource.hubs.MySQL import MySQL
  12
+
  13
+class Model:
  14
+
  15
+   def __init__(self, sqlFileName):
  16
+
  17
+      self.DATAZILLA_DATABASE_NAME     = os.environ["DATAZILLA_DATABASE_NAME"]
  18
+      self.DATAZILLA_DATABASE_USER     = os.environ["DATAZILLA_DATABASE_USER"]
  19
+      self.DATAZILLA_DATABASE_PASSWORD = os.environ["DATAZILLA_DATABASE_PASSWORD"]
  20
+      self.DATAZILLA_DATABASE_HOST     = os.environ["DATAZILLA_DATABASE_HOST"]
  21
+      self.DATAZILLA_DATABASE_PORT     = os.environ["DATAZILLA_DATABASE_PORT"]
  22
+
  23
+      self.sqlFileName = sqlFileName
  24
+
  25
+      try:
  26
+         self.DEBUG = os.environ["DATAZILLA_DEBUG"] is not None
  27
+      except KeyError:
  28
+         self.DEBUG = False
  29
+
  30
+      self.rootPath = os.path.dirname(os.path.abspath(__file__))
  31
+
  32
+      ####
  33
+      #Configuration of datasource hub:
  34
+      #	1 Build the datasource struct
  35
+      # 	2 Add it to the BaseHub
  36
+      #	3 Instantiate a MySQL hub for all derived classes
  37
+      ####
  38
+      dataSource = { self.DATAZILLA_DATABASE_NAME : { "hub":"MySQL",
  39
+                                                      "master_host":{"host":self.DATAZILLA_DATABASE_HOST,
  40
+                                                                     "user":self.DATAZILLA_DATABASE_USER,
  41
+                                                                     "passwd":self.DATAZILLA_DATABASE_PASSWORD},
  42
+                                                                     "default_db":self.DATAZILLA_DATABASE_NAME,
  43
+                                                      "procs": ["%s%s%s" % (self.rootPath,  "/sql/", sqlFileName)]
  44
+                                                    } }
  45
+      BaseHub.addDataSource(dataSource)
  46
+      self.dhub = MySQL(self.DATAZILLA_DATABASE_NAME)
  47
+
  48
+   def setData(self, statement, placeholders):
  49
+
  50
+      self.dhub.execute(proc='graphs.inserts.' + statement,
  51
+                        debug_show=self.DEBUG,
  52
+                        placeholders=placeholders)
  53
+
  54
+      idIter = self.dhub.execute(proc='graphs.selects.get_last_insert_id',
  55
+                                  debug_show=self.DEBUG,
  56
+                                  return_type='iter')
  57
+
  58
+      return idIter.getColumnData('id')
  59
+
  60
+   def isNumber(self, s):
  61
+      try:
  62
+         float(s)
  63
+         return True
  64
+      except ValueError:
  65
+         return False
  66
+
  67
+   def buildReplacement(self, colData):
  68
+
  69
+      rep = "AND "
  70
+
  71
+      for key in colData:
  72
+         if len(colData[key]) > 0:
  73
+            rep += key + ' IN (' + colData[key] + ') AND ' 
  74
+
  75
+      rep = re.sub('AND\s+$', '', rep)  
  76
+
  77
+      return rep
  78
+
  79
+   def disconnect(self):
  80
+      self.dhub.disconnect()
  81
+
0  model/__init__.py
No changes.
459  model/sql/graphs.json
... ...
@@ -0,0 +1,459 @@
  1
+{
  2
+ "views":{ 
  3
+      "test_runs":{
  4
+
  5
+         "sql":"SELECT tr.id AS 'test_run_id',
  6
+                       tr.build_id,
  7
+                       tr.revision,
  8
+                       tr.date_run,
  9
+                       b.processor,
  10
+                       b.build_type,
  11
+                       b.test_build_id,
  12
+                       t.name AS 'Test Name',
  13
+                       br.product,
  14
+                       br.branch,
  15
+                       br.version,
  16
+                       m.name AS 'machine',
  17
+                       m.cpu_speed,
  18
+                       os.name AS 'OS',
  19
+                       os.version AS 'OS Version'
  20
+                 FROM test_run AS tr
  21
+                 LEFT JOIN build AS b ON tr.build_id = b.id
  22
+                 LEFT JOIN test AS t ON tr.test_id = t.id
  23
+                 LEFT JOIN product AS br ON b.product_id = br.id
  24
+                 LEFT JOIN machine AS m ON b.machine_id = m.id
  25
+                 LEFT JOIN operating_system AS os ON b.operating_system_id = os.id
  26
+                 ORDER BY tr.id DESC
  27
+                 LIMIT 1000",
  28
+
  29
+         "host":"master_host"
  30
+
  31
+      },
  32
+      "test_run":{
  33
+
  34
+         "sql":"SELECT tr.id AS 'test_run_id',
  35
+                       tr.build_id,
  36
+                       tr.revision,
  37
+                       tr.date_run,
  38
+                       b.processor,
  39
+                       b.build_type,
  40
+                       b.test_build_id,
  41
+                       t.name AS 'Test Name',
  42
+                       br.product,
  43
+                       br.branch,
  44
+                       br.version,
  45
+                       m.name AS 'machine',
  46
+                       m.cpu_speed,
  47
+                       os.name AS 'OS',
  48
+                       os.version AS 'OS Version'
  49
+                 FROM test_run AS tr
  50
+                 LEFT JOIN build AS b ON tr.build_id = b.id
  51
+                 LEFT JOIN test AS t ON tr.test_id = t.id
  52
+                 LEFT JOIN product AS br ON b.product_id = br.id
  53
+                 LEFT JOIN machine AS m ON b.machine_id = m.id
  54
+                 LEFT JOIN operating_system AS os ON b.operating_system_id = os.id
  55
+                 ORDER BY tr.id DESC
  56
+                 LIMIT 1000",
  57
+
  58
+         "host":"master_host"
  59
+
  60
+      },
  61
+      "test_value":{
  62
+
  63
+         "sql":"SELECT tv.test_run_id,
  64
+                       tv.run_id,
  65
+                       p.url,
  66
+                       v.name,
  67
+                       tv.value
  68
+                FROM test_value AS tv
  69
+                LEFT JOIN pages AS p ON tv.page_id = p.id
  70
+                LEFT JOIN value AS v ON tv.value_id = v.id
  71
+                WHERE tv.test_run_id = ?",
  72
+
  73
+         "host":"master_host"
  74
+      },
  75
+
  76
+      "test_option_values":{
  77
+
  78
+         "sql":"SELECT tov.test_run_id,
  79
+                       o.name,
  80
+                       tov.value
  81
+                FROM test_option_values AS tov
  82
+                LEFT JOIN `option` AS o ON tov.option_id = o.id
  83
+                WHERE tov.test_run_id = ?",
  84
+
  85
+         "host":"master_host"
  86
+      },
  87
+
  88
+      "test_aux_data":{
  89
+
  90
+         "sql":"SELECT tad.test_run_id,
  91
+                       tad.run_id,
  92
+                       ad.name,
  93
+                       tad.numeric_data,
  94
+                       tad.string_data
  95
+                FROM test_aux_data AS tad
  96
+                LEFT JOIN aux_data AS ad ON tad.aux_data_id = ad.id
  97
+                WHERE tad.test_run_id = ?",
  98
+
  99
+         "host":"master_host"
  100
+      }
  101
+ },
  102
+ "inserts":{
  103
+      "set_test":{
  104
+
  105
+         "sql":"INSERT INTO test (`name`, `version`) VALUES (?, ?)",
  106
+
  107
+         "host":"master_host"
  108
+      },
  109
+      "set_operating_system":{
  110
+      
  111
+         "sql":"INSERT INTO operating_system ( name, version ) VALUES (?, ?)",
  112
+
  113
+         "host":"master_host"
  114
+      },
  115
+      "set_aux_data":{
  116
+
  117
+         "sql":"INSERT INTO `aux_data` (`test_id`, `name`) VALUES (?, ?)",
  118
+
  119
+         "host":"master_host"
  120
+      },
  121
+      "set_product_data":{
  122
+         "sql":"INSERT INTO `product` (`product`, `branch`, `version`) VALUES (?, ?, ?)",
  123
+
  124
+         "host":"master_host"
  125
+      },
  126
+      "set_machine_data":{
  127
+
  128
+         "sql":"INSERT INTO `machine` (`name`, `date_added`) VALUES (?, ?)",
  129
+
  130
+         "host":"master_host"
  131
+      },
  132
+      "set_option_data":{
  133
+
  134
+         "sql":"INSERT INTO `options` (`name`) VALUES (?)",
  135
+
  136
+         "host":"master_host"
  137
+      },
  138
+      "set_pages_data":{
  139
+
  140
+         "sql":"INSERT INTO `pages` (`test_id`, `url`) VALUES (?, ?)",
  141
+
  142
+         "host":"master_host"
  143
+      },
  144
+      "set_build_data":{
  145
+
  146
+         "sql":"INSERT INTO `build` (`operating_system_id`,
  147
+                                     `product_id`,
  148
+                                     `machine_id`,
  149
+                                     `test_build_id`,
  150
+                                     `processor`,
  151
+                                     `revision`,
  152
+                                     `build_type`,
  153
+                                     `build_date`)
  154
+                VALUES (?,?,?,?,?,?,?,?)",
  155
+
  156
+         "host":"master_host"
  157
+      },
  158
+      "set_test_run_data":{
  159
+
  160
+         "sql":"INSERT INTO `test_run` (`test_id`,
  161
+                                        `build_id`,
  162
+                                        `revision`,
  163
+                                        `date_run`)
  164
+                VALUES (?,?,?,?)",
  165
+
  166
+         "host":"master_host"
  167
+
  168
+      },
  169
+      "set_test_values":{
  170
+
  171
+         "sql":"INSERT INTO `test_value` (`test_run_id`,
  172
+                                          `run_id`,
  173
+                                          `page_id`,
  174
+                                          `value_id`,
  175
+                                          `value`)
  176
+                VALUES (?,?,?,?,?)",
  177
+
  178
+         "host":"master_host"
  179
+      },
  180
+      "set_aux_values":{
  181
+
  182
+         "sql":"INSERT INTO `test_aux_data` (`test_run_id`,
  183
+                                             `run_id`,
  184
+                                             `aux_data_id`,
  185
+                                             `numeric_data`,
  186
+                                             `string_data`)
  187
+                VALUES (?,?,?,?,?)",
  188
+
  189
+         "host":"master_host"
  190
+      },
  191
+      "set_test_option_values":{
  192
+
  193
+         "sql":"INSERT INTO `test_option_values` (`test_run_id`,
  194
+                                                  `option_id`,
  195
+                                                  `value`)
  196
+                VALUES (?,?,?)",
  197
+
  198
+         "host":"master_host"
  199
+
  200
+      },
  201
+      "set_test_data":{
  202
+
  203
+         "sql":"INSERT INTO `test_data` (`test_run_id`,
  204
+                                         `data`)
  205
+                VALUES (?,?)",
  206
+
  207
+         "host":"master_host"
  208
+      },
  209
+      "set_summary_cache":{
  210
+
  211
+         "sql":"INSERT INTO `summary_cache` (`item_id`, `item_data`, `value`, `date`)
  212
+                VALUES (?, ?, ?, ?)
  213
+                ON DUPLICATE KEY UPDATE `value`=?, `date`=?",
  214
+
  215
+         "host":"master_host"
  216
+      },
  217
+      "set_test_collection":{
  218
+
  219
+         "sql":"INSERT INTO `test_collection` (`name`, `description`)
  220
+                VALUES (?, ?)
  221
+                ON DUPLICATE KEY UPDATE `name`=?",
  222
+
  223
+         "host":"master_host"
  224
+      },
  225
+      "set_test_collection_map":{
  226
+
  227
+         "sql":"INSERT INTO `test_collection_map` (`test_collection_id`, `product_id`)
  228
+                VALUES (?, ?)",
  229
+
  230
+         "host":"master_host"
  231
+      }
  232
+  },
  233
+  "selects":{
  234
+      "get_test_run_summary":{
  235
+
  236
+         "sql":"SELECT tr.id AS 'test_run_id', 
  237
+                       tr.revision,
  238
+                       tr.date_run, 
  239
+                       b.product_id, 
  240
+                       tr.test_id, 
  241
+                       b.operating_system_id,
  242
+                       ROUND( AVG(tv.value), 2 ) AS average, 
  243