rewiring stuff to not use the database

- collections now use static .json files in static/data TODO: cache these in local storage via a collection that somehow intercepts the call to fetch() TODO: once cache is set up, label it with some sort of version. or just expect ppl to clear their cache every once in a while...? - all collections use the same model (there is only one model now) - all collections are very similar now...
rgarcia · Dec 11, 2011 · ba4b377 · ba4b377 · vecter · Dec 11, 2011
1 parent 2ddd04e
commit ba4b377
Show file tree

Hide file tree

Showing 22 changed files with 59 additions and 446 deletions.
diff --git a/README.md b/README.md
@@ -1,26 +1,24 @@
 # Runnning locally
 
-Make sure you MongoDB running on your system. For example, on Ubuntu, type
-
-    apt-get install mongodb
-
-Download the content:
-
-    cd scraper
-    npm install # downloads dependencies for scrapers
-    (cd css-mdn;      ./run.sh)
-    (cd css-cssinfos; ./run.sh)
-    (cd html-mdn;     ./run.sh)
-    (cd js-mdn;       node scrape.js)
-    (cd dom-mdn       node scrape.js)
-
 Run the server:
 
     npm install # downloads dependencies for web server
     node web.js
 
 Open http://localhost:5000/ in your browser.
 
+# Scraper
+
+The `static/data/` directory contains our scrape of the sites we get content from. Right now we don't update this very often. If you want the most up-to-date content, you can run the scrapers:
+
+    cd scraper
+    npm install # downloads dependencies for scrapers
+    (cd css-mdn;  node scrape.js)
+    (cd html-mdn; node scrape.js)
+    (cd js-mdn;   node scrape.js)
+    (cd dom-mdn;  node scrape.js)
+    (cd jquery;   node scrape.js)
+
 # Notes
 
 How to use the r.js optimizer:

diff --git a/app.js b/app.js
@@ -1,11 +1,11 @@
 define([
   'express',
-  'mongoose',
+  // 'mongoose',
-  'controllers/loader',
+  // 'controllers/loader',
   'module',
   'path',
   './config'
-], function (express, mongoose, controllerLoader, module, path, config) {
+], function (express,/* mongoose, controllerLoader,*/ module, path, config) {
 
   var app = null;
 
@@ -20,20 +20,20 @@ define([
       app.listen(config.app_port);
 
       app.configure(function() {
-        var db = mongoose.connect(config.mongo_uri, function(err) {
+        // var db = mongoose.connect(config.mongo_uri, function(err) {
-          if (err)
+        //   if (err)
-            throw err;
+        //     throw err;
-          else
+        //   else
-            console.log('connected to ' + config.mongo_uri);
+        //     console.log('connected to ' + config.mongo_uri);
-        });
+        // });
         app.use(express.logger({ format: ':method :url :status' }));
         var filename = module.uri;
         var staticDir = path.dirname(filename) + '/static';
         console.log('initializing static: ' + staticDir);
         app.use(express.static(staticDir));
         app.use(express.bodyParser());
         app.use(express.methodOverride());
-        controllerLoader.bootControllers(app);
+        //controllerLoader.bootControllers(app);
         console.log('instacss version now running on port ' + config.app_port);
       });
     },

diff --git a/config.js b/config.js
@@ -9,8 +9,7 @@ define([], function() {
     environment:  global.process.env.NODE_ENV || 'development',
 
     // MONGOLAB_URI is provided by heroku mongolab add-on
-    mongo_uri: global.process.env.MONGOLAB_URI || 'mongodb://127.0.0.1/instacss',
+    //mongo_uri: global.process.env.MONGOLAB_URI || 'mongodb://127.0.0.1/instacss',
 
   };
 });
-
diff --git a/models/cssprop.js b/models/cssprop.js
diff --git a/models/jqentry.js b/models/jqentry.js
diff --git a/models/mdndomobj.js b/models/mdndomobj.js
diff --git a/models/mdnhtmlelement.js b/models/mdnhtmlelement.js
diff --git a/models/mdnjsobj.js b/models/mdnjsobj.js
diff --git a/models/mozdevcssprop.js b/models/mozdevcssprop.js
diff --git a/models/phpext.js b/models/phpext.js
diff --git a/static/js/collections/jqentries.js b/static/js/collections/jqentries.js
@@ -2,18 +2,18 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/jqentry'
+  'models/sectionscrape'
-], function($, _, Backbone, JQEntry) {
+], function($, _, Backbone, SectionScrape) {
 
   var jqEntryPattern = new RegExp("^(\\W*)(.+)$");
 
   var JQEntry = Backbone.Collection.extend({
-    url: '/jqentry',
+    url: '/data/jquery.json',
-    model: JQEntry,
+    model: SectionScrape,
 
 
     comparator: function(model) {
-      var title = model.get('fullTitle');
+      var title = model.get('title');
       var results = jqEntryPattern.exec(title);
       var prefix = results[1];
       var name   = results[2];
@@ -30,4 +30,3 @@ define([
 
   return JQEntry;
 });
-
diff --git a/static/js/collections/mdndomobjs.js b/static/js/collections/mdndomobjs.js
@@ -2,18 +2,17 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/mdndomobj'
+  'models/sectionscrape'
-], function($, _, Backbone, MDNDomObj) {
+], function($, _, Backbone, SectionScrape) {
 
   var MDNDomObj = Backbone.Collection.extend({
-    url: '/mdndomobj',
+    url: '/data/dom-mdn.json',
-    model: MDNDomObj,
+    model: SectionScrape,
 
     comparator: function(model) {
-      return model.get('fullTitle');
+      return model.get('title');
     },
   });
 
   return MDNDomObj;
 });
-
diff --git a/static/js/collections/mdnhtmlelements.js b/static/js/collections/mdnhtmlelements.js
@@ -2,16 +2,16 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/mdnhtmlelement'
+  'models/sectionscrape'
-], function($, _, Backbone, MDNHtmlElement) {
+], function($, _, Backbone, SectionScrape) {
 
   // Handle the following cases:
   //  <xyz>
   var htmlElementPattern = new RegExp("^(\\W*)(.+)$");
 
   var MDNHtmlElemnts = Backbone.Collection.extend({
-    url: '/mdnhtmlelement',
+    url: '/data/html-mdn.json',
-    model: MDNHtmlElement,
+    model: SectionScrape,
 
   comparator: function(model) {
     var title = model.get('title');

diff --git a/static/js/collections/mdnjsobjs.js b/static/js/collections/mdnjsobjs.js
@@ -2,18 +2,18 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/mdnjsobj'
+  'models/sectionscrape'
-], function($, _, Backbone, MDNJsObj) {
+], function($, _, Backbone, SectionScrape) {
 
   var MDNJsObj = Backbone.Collection.extend({
-    url: '/mdnjsobj',
+    url: '/data/js-mdn.json',
-    model: MDNJsObj,
+    model: SectionScrape,
 
     comparator: function(model) {
-      return model.get('fullTitle');
+      return model.get('title');
     },
+
   });
 
   return MDNJsObj;
 });
-
diff --git a/static/js/collections/mozdevcssprops.js b/static/js/collections/mozdevcssprops.js
@@ -2,8 +2,8 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/mozdevcssprop'
+  'models/sectionscrape'
-], function($, _, Backbone, MozDevCSSProp) {
+], function($, _, Backbone, SectionScrape) {
 
   // Handle the following cases:
   //  -xyz
@@ -15,17 +15,17 @@ define([
   var cssPropsPattern = new RegExp("^(\\W*)(.+)$");
 
   var MozDevCSSProps = Backbone.Collection.extend({
-    url: '/mozdevcssprop',
+    url: '/data/css-mdn.json',
-    model: MozDevCSSProp,
+    model: SectionScrape,
 
-  comparator: function(model) {
+    comparator: function(model) {
-    var title = model.get('title');
+      var title = model.get('title');
-    var results = cssPropsPattern.exec(title);
+      var results = cssPropsPattern.exec(title);
-    var prefix = results[1];
+      var prefix = results[1];
-    var name   = results[2].toLowerCase();
+      var name   = results[2].toLowerCase();
 
-    return prefix ? ('1' + title) : ('0' + name);
+      return prefix ? ('1' + title) : ('0' + name);
-  },
+    },
 
   });
 

diff --git a/static/js/collections/phpexts.js b/static/js/collections/phpexts.js
@@ -2,18 +2,17 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  'models/phpext'
+  'models/SectionScrape'
-], function($, _, Backbone, PhpExt) {
+], function($, _, Backbone, SectionScrape) {
 
   var PhpExt = Backbone.Collection.extend({
-    url: '/phpext',
+    url: '/data/php.json',
-    model: PhpExt,
+    model: SectionScrape,
 
     comparator: function(model) {
-      return model.get('fullTitle').toLowerCase();
+      return model.get('title').toLowerCase();
     },
   });
 
   return PhpExt;
 });
-