Permalink
Browse files

first commit

  • Loading branch information...
jeads committed Apr 26, 2012
0 parents commit 42be953cf10b2074886778c4560ded1f14ecea74
Showing with 53,504 additions and 0 deletions.
  1. +5 −0 LICENSE.txt
  2. +113 −0 README.md
  3. 0 __init__.py
  4. 0 controller/__init__.py
  5. 0 controller/admin/__init__.py
  6. +90 −0 controller/admin/populate_summary_cache.py
  7. +44 −0 controller/admin/populate_test_collections.py
  8. +510 −0 model/DatazillaModel.py
  9. +81 −0 model/Model.py
  10. 0 model/__init__.py
  11. +459 −0 model/sql/graphs.json
  12. +57 −0 model/t/DatazillaModelTest.py
  13. 0 model/t/__init__.py
  14. 0 webapp/__init__.py
  15. 0 webapp/apps/__init__.py
  16. 0 webapp/apps/datazilla/__init__.py
  17. 0 webapp/apps/datazilla/filters/__init__.py
  18. 0 webapp/apps/datazilla/filters/templatetags/__init__.py
  19. +60 −0 webapp/apps/datazilla/filters/templatetags/dv_unorderedlist.py
  20. 0 webapp/apps/datazilla/management/__init__.py
  21. 0 webapp/apps/datazilla/management/commands/__init__.py
  22. +83 −0 webapp/apps/datazilla/management/commands/build_nav.py
  23. +3 −0 webapp/apps/datazilla/models.py
  24. +16 −0 webapp/apps/datazilla/tests.py
  25. +9 −0 webapp/apps/datazilla/urls.py
  26. +310 −0 webapp/apps/datazilla/views.py
  27. +15 −0 webapp/conf/bin/datazilla
  28. +6 −0 webapp/conf/bin/start_datazilla
  29. +19 −0 webapp/conf/etc/nginx/conf.d/datazilla.conf
  30. +24 −0 webapp/conf/etc/nginx/fastcgi_params
  31. +101 −0 webapp/conf/etc/rc.d/init.d/datazilla
  32. +13 −0 webapp/conf/etc/sysconfig/datazilla
  33. +14 −0 webapp/manage.py
  34. BIN webapp/media/css/Aristo/images/bg_fallback.png
  35. BIN webapp/media/css/Aristo/images/icon_sprite.png
  36. BIN webapp/media/css/Aristo/images/progress_bar.gif
  37. BIN webapp/media/css/Aristo/images/slider_handles.png
  38. BIN webapp/media/css/Aristo/images/ui-icons_222222_256x240.png
  39. BIN webapp/media/css/Aristo/images/ui-icons_454545_256x240.png
  40. +756 −0 webapp/media/css/Aristo/jquery-ui-1.8.7.custom.css
  41. +108 −0 webapp/media/css/ColVisAlt.css
  42. +216 −0 webapp/media/css/common.css
  43. +312 −0 webapp/media/css/datatables.css
  44. +397 −0 webapp/media/css/dataviews.css
  45. +114 −0 webapp/media/css/fg.menu.css
  46. +53 −0 webapp/media/html/control_panels/named_fields.html
  47. +81 −0 webapp/media/html/control_panels/test_selector.html
  48. +28 −0 webapp/media/html/nav_menu.html
  49. BIN webapp/media/images/back_disabled.jpg
  50. BIN webapp/media/images/back_enabled.jpg
  51. BIN webapp/media/images/background.png
  52. BIN webapp/media/images/bhview_panel.png
  53. BIN webapp/media/images/col_button.png
  54. BIN webapp/media/images/dino3.png
  55. BIN webapp/media/images/forward_disabled.jpg
  56. BIN webapp/media/images/forward_enabled.jpg
  57. BIN webapp/media/images/sort_asc.png
  58. BIN webapp/media/images/sort_asc_disabled.png
  59. BIN webapp/media/images/sort_both.png
  60. BIN webapp/media/images/sort_desc.png
  61. BIN webapp/media/images/sort_desc_disabled.png
  62. BIN webapp/media/images/spinner.gif
  63. BIN webapp/media/images/title.png
  64. +105 −0 webapp/media/js/data_views/Bases.js
  65. +176 −0 webapp/media/js/data_views/ConnectionsComponent.js
  66. +828 −0 webapp/media/js/data_views/DataAdapterCollection.js
  67. +493 −0 webapp/media/js/data_views/DataViewCollection.js
  68. +1,834 −0 webapp/media/js/data_views/DataViewComponent.js
  69. +129 −0 webapp/media/js/data_views/DataViewPage.js
  70. +28 −0 webapp/media/js/data_views/HelpPage.js
  71. +1,657 −0 webapp/media/js/data_views/VisualizationCollection.js
  72. +386 −0 webapp/media/js/plugins/box.js
  73. +9,342 −0 webapp/media/js/plugins/d3.v2.js
  74. +4 −0 webapp/media/js/plugins/d3.v2.min.js
  75. +1,020 −0 webapp/media/js/plugins/datatables/ColReorderWithResize.js
  76. +875 −0 webapp/media/js/plugins/datatables/ColVis.js
  77. +7,465 −0 webapp/media/js/plugins/datatables/jquery.dataTables.js
  78. +151 −0 webapp/media/js/plugins/datatables/jquery.dataTables.min.js
  79. +639 −0 webapp/media/js/plugins/fg.menu.js
  80. +1,201 −0 webapp/media/js/plugins/flot-0.7/API.txt
  81. +76 −0 webapp/media/js/plugins/flot-0.7/FAQ.txt
  82. +22 −0 webapp/media/js/plugins/flot-0.7/LICENSE.txt
  83. +9 −0 webapp/media/js/plugins/flot-0.7/Makefile
  84. +508 −0 webapp/media/js/plugins/flot-0.7/NEWS.txt
  85. +137 −0 webapp/media/js/plugins/flot-0.7/PLUGINS.txt
  86. +90 −0 webapp/media/js/plugins/flot-0.7/README.txt
  87. +143 −0 webapp/media/js/plugins/flot-0.7/examples/ajax.html
  88. +75 −0 webapp/media/js/plugins/flot-0.7/examples/annotating.html
  89. BIN webapp/media/js/plugins/flot-0.7/examples/arrow-down.gif
  90. BIN webapp/media/js/plugins/flot-0.7/examples/arrow-left.gif
  91. BIN webapp/media/js/plugins/flot-0.7/examples/arrow-right.gif
  92. BIN webapp/media/js/plugins/flot-0.7/examples/arrow-up.gif
  93. +38 −0 webapp/media/js/plugins/flot-0.7/examples/basic.html
  94. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-1.json
  95. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-2.json
  96. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-3.json
  97. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-4.json
  98. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth-5.json
  99. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-eu-gdp-growth.json
  100. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-japan-gdp-growth.json
  101. +4 −0 webapp/media/js/plugins/flot-0.7/examples/data-usa-gdp-growth.json
  102. +75 −0 webapp/media/js/plugins/flot-0.7/examples/graph-types.html
  103. BIN webapp/media/js/plugins/flot-0.7/examples/hs-2004-27-a-large_web.jpg
  104. +45 −0 webapp/media/js/plugins/flot-0.7/examples/image.html
  105. +44 −0 webapp/media/js/plugins/flot-0.7/examples/index.html
  106. +97 −0 webapp/media/js/plugins/flot-0.7/examples/interacting-axes.html
  107. +93 −0 webapp/media/js/plugins/flot-0.7/examples/interacting.html
  108. +6 −0 webapp/media/js/plugins/flot-0.7/examples/layout.css
  109. +60 −0 webapp/media/js/plugins/flot-0.7/examples/multiple-axes.html
  110. +118 −0 webapp/media/js/plugins/flot-0.7/examples/navigate.html
  111. +57 −0 webapp/media/js/plugins/flot-0.7/examples/percentiles.html
  112. +756 −0 webapp/media/js/plugins/flot-0.7/examples/pie.html
  113. +83 −0 webapp/media/js/plugins/flot-0.7/examples/realtime.html
  114. +61 −0 webapp/media/js/plugins/flot-0.7/examples/resize.html
  115. +114 −0 webapp/media/js/plugins/flot-0.7/examples/selection.html
  116. +61 −0 webapp/media/js/plugins/flot-0.7/examples/setting-options.html
  117. +77 −0 webapp/media/js/plugins/flot-0.7/examples/stacking.html
  118. +49 −0 webapp/media/js/plugins/flot-0.7/examples/symbols.html
  119. +54 −0 webapp/media/js/plugins/flot-0.7/examples/thresholding.html
  120. +71 −0 webapp/media/js/plugins/flot-0.7/examples/time.html
  121. +95 −0 webapp/media/js/plugins/flot-0.7/examples/tracking.html
  122. +98 −0 webapp/media/js/plugins/flot-0.7/examples/turning-series.html
  123. +90 −0 webapp/media/js/plugins/flot-0.7/examples/visitors.html
  124. +98 −0 webapp/media/js/plugins/flot-0.7/examples/zooming.html
  125. +1,427 −0 webapp/media/js/plugins/flot-0.7/excanvas.js
  126. +1 −0 webapp/media/js/plugins/flot-0.7/excanvas.min.js
  127. +179 −0 webapp/media/js/plugins/flot-0.7/jquery.colorhelpers.js
  128. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.colorhelpers.min.js
  129. +401 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.axislabels.js
  130. +167 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.crosshair.js
  131. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.crosshair.min.js
  132. +183 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.fillbetween.js
  133. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.fillbetween.min.js
  134. +238 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.image.js
  135. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.image.min.js
  136. +2,599 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.js
  137. +6 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.min.js
  138. +336 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.navigate.js
  139. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.navigate.min.js
  140. +187 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.orderBars.js
  141. +750 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.pie.js
  142. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.pie.min.js
  143. +60 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.resize.js
  144. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.resize.min.js
  145. +344 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.selection.js
  146. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.selection.min.js
  147. +184 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.stack.js
  148. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.stack.min.js
  149. +70 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.symbol.js
  150. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.symbol.min.js
  151. +103 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.threshold.js
  152. +1 −0 webapp/media/js/plugins/flot-0.7/jquery.flot.threshold.min.js
  153. +8,316 −0 webapp/media/js/plugins/flot-0.7/jquery.js
  154. +23 −0 webapp/media/js/plugins/flot-0.7/jquery.min.js
  155. +4 −0 webapp/media/js/plugins/jquery-1.6.3.min.js
  156. +784 −0 webapp/media/js/plugins/jquery-ui-1.8.13.custom.min.js
  157. +143 −0 webapp/media/js/plugins/jquery.dataTables.min.js
  158. +714 −0 webapp/media/js/plugins/jquery.flot.text.js
  159. +159 −0 webapp/media/js/plugins/jquery.url.js
  160. +480 −0 webapp/media/js/plugins/json2.js
  161. +39 −0 webapp/media/js/plugins/moo4q/Source/Class.Mutators.jQuery.js
  162. +1,031 −0 webapp/media/js/plugins/mootools-core-1.4.0.js
  163. +26 −0 webapp/media/js/plugins/underscore-min.js
  164. +183 −0 webapp/settings.py
  165. +128 −0 webapp/templates/data/views.json
  166. +1 −0 webapp/templates/graphs.navlookup.html
  167. +200 −0 webapp/templates/graphs.views.html
  168. +145 −0 webapp/templates/help/dataview.generic.help.html
  169. +20 −0 webapp/urls.py
@@ -0,0 +1,5 @@
+#####
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+#####
113 README.md
@@ -0,0 +1,113 @@
+#Datazilla
+Datazilla is a generic system for managing and visualizing data. The fundamental unit of data display in the user interface is called a data view. Data views can display data in any number of ways: tabular or graphical. Data views can also send signals to one another enabling the user to maintain visual context across multiple graphical displays of different data types. Each data view shares a toolbar that abstracts navigation, data presentation controls, and visual presentation. A prototype of datazilla was first developed in an application called [bughunter] [1].
+
+This project includes a model, webservice, and web based user interface, and eventually it will support a local development environment.
+
+This is a work in progress and will likely see a number of structural changes. It is currently being developed to manage [Talos] [2] test data, a performance testing framework developed by mozilla for testing software products.
+
+[1]: https://wiki.mozilla.org/Auto-tools/Projects/BugHunter "bughunter"
+[2]: https://wiki.mozilla.org/Buildbot/Talos "Talos"
+
+##Architecture
+At a top level datazilla can be described with three different parts: model, webservice, and UI.
+
+All environment information is stored in datazilla/webapp/conf/etc/sysconfig/datazilla. Appropriate system information for the system should be added to this file and then it copy to /etc/sysconfig/datazilla. It needs to be source'd before running any component of the system. Most of the environment variables are self explanatory.
+
+The environment variable called DATAZILLA_DEBUG, when set to true, causes all scripts and webservice methods to write out the full SQL, execution time, and host name that are being executed at runtime. This is handy for debugging any component in the system.
+
+###Model
+The datazilla model classes rely on a module called [datasource] [3]. This module encapsulates SQL manipulation. All of the SQL used by the system is stored in a JSON file format in found in /datazilla/model/sql. There can be any number of SQL files stored in this format. The JSON structure allows a host type to be associated with each SQL statement.
+
+[3]: https://github.com/jeads/datasource "datasource"
+
+###UI
+
+####Building the Navigation Menu And Defining Data Views
+New data views and collections of dataviews can be defined in The navigation menu
+
+####Building the Cached Summaries
+
+
+####JS
+
+#####Class Structures
+The javascript that implements the user interface is constructed using a page/component/collection pattern thingy... whatever that means. This pattern was found to be very useful in separating out the required functionality, below is a brief definition of what that means in the data view UI architecture.
+
+######Page
+Manages the DOM ready event, implements any top level initialization that's required for the page. An instance of the page class is the only global variable that other components can access, if they're playing nice. The page class instance is responsible for instantiating components and storing them in attributes. The page class also holds any data structures that need to be globally accessible to component classes.
+
+######Component
+Contains the public interface of the component. A component can encapsulate any functional subset/unit provided in a page. The component will typically have an instance of a View and Model class. The component class is also responsible for any required event binding.
+
+######View
+A component's view class manages interfacing with the DOM. Any CSS class names or HTML id's are defined as attributes of the view. Any HTML element modification is controlled with this class.
+
+######Model
+A component's model manages any asynchronous data retrieval and large data structure manipulation.
+
+######Collection
+A class for managing a collection of Components or classes of any type. A collection can also have a model/view if appropriate.
+
+######Client Application (datazilla/webapp/media/js/data_views)
+This is not a complete file or class listing but is intended to give a top level description of the design pattern thingy of the data view javascript and what the basic functional responsibility of the pages/components/collections are.
+
+#######DataViewPage.js
+DataViewPage Class - Manages the DOM ready event, component initialization, and retrieval of the views.json structure that is used by different components.
+
+#######Bases.js
+Design Pattern Base Classes - Contains the base classes for Page, Component, Model, View etc...
+
+#######DataViewComponent.js
+DataViewComponent Class - Encapsulates the behavior of a single data view using a model/view and provides a public interface for data view functionality. Manages event binding and registration.
+
+DVViewView Class - Encapsulates all DOM interaction required by a data view.
+
+BHViewModel Class - Encapsulates asynchronous server communication and data structure manipulation/retrieval.
+
+#######DataViewCollection.js
+
+DataViewCollection Class - Manages operations on a collection of data views using a model/view including instantiating view collections.
+
+DataViewCollectionView Class - Encapsulates all DOM interaction required by the collection.
+
+DataViewCollectionModel Class - Provides an interface to the datastructures holding all data views and their associated parent/child relationships.
+
+#######DataAdapterCollection.js
+
+DataAdapterCollection Class - Collection of DataViewAdapter class instances.
+
+BHViewAdapter Class - Base class for all BHViewAdapters. Manages shared view idiosyncratic behavior like what fields go in the control panel and how to populate/retrieve them for signaling behavior.
+
+CrashesAdapter Class - Derived class of BHViewAdapter. Encapsulates unique behavior for crash data views.
+
+UrlAdapter Class - Derived class of BHViewAdapter. Encapsulates unique behavior for views containing URL summaries.
+
+##Installation
+1. Add system info to appropriate files in datazilla/webapp/conf/etc. Copy the files to there appropriate location under /etc.
+
+2. Start the datazilla, nginx, and memcached services. There is a startup script for the datazilla and nginx services in datazilla/webapp/conf/bin.
+
+##RHEL6 Configuration
+
+This configuration was done on a RHEL6 VM.
+
+1. cat /etc/redhat-release to get the correct version of EPEL
+
+2. rpm -Uvh http://download.fedora.redhat.com/pub/epel/6/i386/epel-release-6-5.noarch.rpm
+
+3. yum install nginx fcgi Django Django-doc python-docutils MySQL-python python-flup
+
+4. yum install python-setuptools spawn-fcgi
+
+5. yum install mysql.x86_64 mysql-server.x86_64 mysql-devel.x86_64
+
+6. yum install git
+
+7. git clone https://github.com/jeads/datasource, python setup.py install
+
+8. yum install memcached.x86_64 python-memcached.noarch
+
+9. Modify the contents of files in the datazilla/webapp/conf/etc directory to meet the needs of
+ your system and then copy the files to their corresponding locations under /etc.
+
+
No changes.
No changes.
No changes.
@@ -0,0 +1,90 @@
+import os
+import sys
+import time
+import datetime
+import json
+import memcache
+import zlib
+
+from optparse import OptionParser
+from datazilla.model.DatazillaModel import DatazillaModel
+
+"""
+This script builds the test run summary data structure for
+a 7 and 30 day period interval for every product/branch/version.
+
+These data structures are stored in the summary_cache table. They
+need to persist if the memcache goes down, they take several minutes
+to generate. As the quantity of data grows this will likely take
+significantly longer.
+"""
+
+def cacheTestSummaries():
+
+ gm = DatazillaModel('graphs.json')
+ dataIter = gm.getAllSummaryCacheData()
+
+ mc = memcache.Client([os.environ["DATAZILLA_MEMCACHED"]], debug=0)
+
+ for d in dataIter:
+ for data in d:
+ key = DatazillaModel.getCacheKey( data['item_id'], data['item_data'] )
+ rv = mc.set(key, zlib.compress( data['value'] ))
+ if not rv:
+ sys.stderr.write("ERROR: Failed to store object in memcache: %s, %s\n" % ( str(data['item_id']), data['item_data'] ) )
+
+ gm.disconnect()
+
+def buildTestSummaries():
+
+ gm = DatazillaModel('graphs.json')
+
+ timeRanges = DatazillaModel.getTimeRanges()
+
+ products = gm.getProducts()
+
+ for productName in products:
+
+ for tr in ['days_7', 'days_30']:
+
+ table = gm.getTestRunSummary(str( timeRanges[tr]['start']),
+ str( timeRanges[tr]['stop']),
+ [ products[ productName ] ],
+ [],
+ [])
+
+ jsonData = json.dumps( table )
+
+ gm.setSummaryCache( products[ productName ], tr, jsonData )
+
+ gm.disconnect()
+
+if __name__ == '__main__':
+
+ usage = """usage: %prog [options] --build --cache --verbose"""
+ parser = OptionParser(usage=usage)
+
+ parser.add_option('-b',
+ '--build',
+ action='store_true',
+ dest='build',
+ default=False,
+ type=None,
+ help="Build the test run summaries and store them in the database.")
+
+ parser.add_option('-c',
+ '--cache',
+ action='store_true',
+ dest='cache',
+ default=False,
+ type=None,
+ help="Update the test run summaries in memcached")
+
+ (options, args) = parser.parse_args()
+
+ if options.build:
+ buildTestSummaries()
+
+ if options.cache:
+ cacheTestSummaries()
+
@@ -0,0 +1,44 @@
+import os
+import sys
+
+from optparse import OptionParser
+from datazilla.model.DatazillaModel import DatazillaModel
+
+def loadTestCollection():
+
+ gm = DatazillaModel('graphs.json')
+
+ products = gm.getProducts('id')
+
+ for productName in products:
+
+ if products[ productName ]['product'] and \
+ products[ productName ]['version'] and \
+ products[ productName ]['branch']:
+
+ name = "%s %s %s" % (products[ productName ]['product'],
+ products[ productName ]['version'],
+ products[ productName ]['branch'])
+
+ id = gm.setData('set_test_collection', [ name, "", name ])
+ gm.setData('set_test_collection_map', [ id, products[ productName ]['id'] ])
+
+ gm.disconnect()
+
+if __name__ == '__main__':
+
+ usage = """usage: %prog [options] --load --verbose"""
+ parser = OptionParser(usage=usage)
+
+ parser.add_option('-l',
+ '--load',
+ action='store_true',
+ dest='load',
+ default=False,
+ type=None,
+ help="Identitfy new product branches and add them as test collections.")
+
+ (options, args) = parser.parse_args()
+
+ if options.load:
+ loadTestCollection()
Oops, something went wrong.

0 comments on commit 42be953

Please sign in to comment.