From ba4b377fd9bc87ef55a81f040d149e14a8b08438 Mon Sep 17 00:00:00 2001 From: Rafael Garcia Date: Sun, 11 Dec 2011 12:17:58 -0600 Subject: [PATCH] rewiring stuff to not use the database - collections now use static .json files in static/data TODO: cache these in local storage via a collection that somehow intercepts the call to fetch() TODO: once cache is set up, label it with some sort of version. or just expect ppl to clear their cache every once in a while...? - all collections use the same model (there is only one model now) - all collections are very similar now... --- README.md | 26 +++++++------- app.js | 20 +++++------ config.js | 3 +- models/cssprop.js | 31 ----------------- models/jqentry.js | 17 ---------- models/mdndomobj.js | 17 ---------- models/mdnhtmlelement.js | 15 --------- models/mdnjsobj.js | 17 ---------- models/mozdevcssprop.js | 15 --------- models/phpext.js | 17 ---------- static/js/collections/jqentries.js | 11 +++--- static/js/collections/mdndomobjs.js | 11 +++--- static/js/collections/mdnhtmlelements.js | 8 ++--- static/js/collections/mdnjsobjs.js | 12 +++---- static/js/collections/mozdevcssprops.js | 22 ++++++------ static/js/collections/phpexts.js | 11 +++--- static/js/models/jqentry.js | 42 ----------------------- static/js/models/mdndomobj.js | 43 ------------------------ static/js/models/mdnhtmlelement.js | 41 ---------------------- static/js/models/mdnjsobj.js | 43 ------------------------ static/js/models/mozdevcssprop.js | 41 ---------------------- static/js/models/phpext.js | 42 ----------------------- 22 files changed, 59 insertions(+), 446 deletions(-) delete mode 100644 models/cssprop.js delete mode 100644 models/jqentry.js delete mode 100644 models/mdndomobj.js delete mode 100644 models/mdnhtmlelement.js delete mode 100644 models/mdnjsobj.js delete mode 100644 models/mozdevcssprop.js delete mode 100644 models/phpext.js delete mode 100644 static/js/models/jqentry.js delete mode 100644 static/js/models/mdndomobj.js delete mode 100644 static/js/models/mdnhtmlelement.js delete mode 100644 static/js/models/mdnjsobj.js delete mode 100644 static/js/models/mozdevcssprop.js delete mode 100644 static/js/models/phpext.js diff --git a/README.md b/README.md index 87d218a..b9d5baf 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,5 @@ # Runnning locally -Make sure you MongoDB running on your system. For example, on Ubuntu, type - - apt-get install mongodb - -Download the content: - - cd scraper - npm install # downloads dependencies for scrapers - (cd css-mdn; ./run.sh) - (cd css-cssinfos; ./run.sh) - (cd html-mdn; ./run.sh) - (cd js-mdn; node scrape.js) - (cd dom-mdn node scrape.js) - Run the server: npm install # downloads dependencies for web server @@ -21,6 +7,18 @@ Run the server: Open http://localhost:5000/ in your browser. +# Scraper + +The `static/data/` directory contains our scrape of the sites we get content from. Right now we don't update this very often. If you want the most up-to-date content, you can run the scrapers: + + cd scraper + npm install # downloads dependencies for scrapers + (cd css-mdn; node scrape.js) + (cd html-mdn; node scrape.js) + (cd js-mdn; node scrape.js) + (cd dom-mdn; node scrape.js) + (cd jquery; node scrape.js) + # Notes How to use the r.js optimizer: diff --git a/app.js b/app.js index 598b95c..619dd34 100644 --- a/app.js +++ b/app.js @@ -1,11 +1,11 @@ define([ 'express', - 'mongoose', - 'controllers/loader', + // 'mongoose', + // 'controllers/loader', 'module', 'path', './config' -], function (express, mongoose, controllerLoader, module, path, config) { +], function (express,/* mongoose, controllerLoader,*/ module, path, config) { var app = null; @@ -20,12 +20,12 @@ define([ app.listen(config.app_port); app.configure(function() { - var db = mongoose.connect(config.mongo_uri, function(err) { - if (err) - throw err; - else - console.log('connected to ' + config.mongo_uri); - }); + // var db = mongoose.connect(config.mongo_uri, function(err) { + // if (err) + // throw err; + // else + // console.log('connected to ' + config.mongo_uri); + // }); app.use(express.logger({ format: ':method :url :status' })); var filename = module.uri; var staticDir = path.dirname(filename) + '/static'; @@ -33,7 +33,7 @@ define([ app.use(express.static(staticDir)); app.use(express.bodyParser()); app.use(express.methodOverride()); - controllerLoader.bootControllers(app); + //controllerLoader.bootControllers(app); console.log('instacss version now running on port ' + config.app_port); }); }, diff --git a/config.js b/config.js index 640a1ef..05a7b7a 100644 --- a/config.js +++ b/config.js @@ -9,8 +9,7 @@ define([], function() { environment: global.process.env.NODE_ENV || 'development', // MONGOLAB_URI is provided by heroku mongolab add-on - mongo_uri: global.process.env.MONGOLAB_URI || 'mongodb://127.0.0.1/instacss', + //mongo_uri: global.process.env.MONGOLAB_URI || 'mongodb://127.0.0.1/instacss', }; }); - diff --git a/models/cssprop.js b/models/cssprop.js deleted file mode 100644 index 728a146..0000000 --- a/models/cssprop.js +++ /dev/null @@ -1,31 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var PropValue = new Schema({ - value : String, - description : String, - }); - - var Example = new Schema({ - description : String, - code : String, - }); - - var CSSProp = new Schema({ - id : ObjectId, - name : { type: String, required: true }, - srcUrl : { type: String, required: true }, - description : { type: String, required: true }, - version : { type: String, required: true }, - defaultValue : String, - inherited : String, - jsSyntax : String, - values : [PropValue], - examples : [Example] - }); - - return mongoose.model('CSSProp',CSSProp); - -}); diff --git a/models/jqentry.js b/models/jqentry.js deleted file mode 100644 index 97513d0..0000000 --- a/models/jqentry.js +++ /dev/null @@ -1,17 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var JQEntry = new Schema({ - id : ObjectId, - title : { type: String, required: true }, // e.g. length - fullTitle : { type: String, required: true }, // e.g. Array.length - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('JQEntry', JQEntry); - -}); - diff --git a/models/mdndomobj.js b/models/mdndomobj.js deleted file mode 100644 index 4608e2d..0000000 --- a/models/mdndomobj.js +++ /dev/null @@ -1,17 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var MDNDomObj = new Schema({ - id : ObjectId, - title : { type: String, required: true }, // e.g. length - fullTitle : { type: String, required: true }, // e.g. Array.length - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('MDNDomObj', MDNDomObj); - -}); - diff --git a/models/mdnhtmlelement.js b/models/mdnhtmlelement.js deleted file mode 100644 index dec81e9..0000000 --- a/models/mdnhtmlelement.js +++ /dev/null @@ -1,15 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var MDNHtmlElement = new Schema({ - id : ObjectId, - title : { type: String, required: true }, - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('MDNHtmlElement', MDNHtmlElement); - -}); diff --git a/models/mdnjsobj.js b/models/mdnjsobj.js deleted file mode 100644 index f7719c7..0000000 --- a/models/mdnjsobj.js +++ /dev/null @@ -1,17 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var MDNJsObj = new Schema({ - id : ObjectId, - title : { type: String, required: true }, // e.g. length - fullTitle : { type: String, required: true }, // e.g. Array.length - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('MDNJsObj', MDNJsObj); - -}); - diff --git a/models/mozdevcssprop.js b/models/mozdevcssprop.js deleted file mode 100644 index 494ca0c..0000000 --- a/models/mozdevcssprop.js +++ /dev/null @@ -1,15 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var MozDevCSSProp = new Schema({ - id : ObjectId, - title : { type: String, required: true }, - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('MozDevCSSProp',MozDevCSSProp); - -}); diff --git a/models/phpext.js b/models/phpext.js deleted file mode 100644 index 88ecefb..0000000 --- a/models/phpext.js +++ /dev/null @@ -1,17 +0,0 @@ -define(['mongoose'], function(mongoose) { - - var Schema = mongoose.Schema; - var ObjectId = Schema.ObjectId; - - var PHPExtension = new Schema({ - id : ObjectId, - title : { type: String, required: true }, // e.g. length - fullTitle : { type: String, required: true }, // e.g. Array.length - sectionNames : [String], // ordering important - sectionHTMLs : [String] - }); - - return mongoose.model('PHPExtension', PHPExtension); - -}); - diff --git a/static/js/collections/jqentries.js b/static/js/collections/jqentries.js index 1e84a6f..a695a38 100644 --- a/static/js/collections/jqentries.js +++ b/static/js/collections/jqentries.js @@ -2,18 +2,18 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/jqentry' -], function($, _, Backbone, JQEntry) { + 'models/sectionscrape' +], function($, _, Backbone, SectionScrape) { var jqEntryPattern = new RegExp("^(\\W*)(.+)$"); var JQEntry = Backbone.Collection.extend({ - url: '/jqentry', - model: JQEntry, + url: '/data/jquery.json', + model: SectionScrape, comparator: function(model) { - var title = model.get('fullTitle'); + var title = model.get('title'); var results = jqEntryPattern.exec(title); var prefix = results[1]; var name = results[2]; @@ -30,4 +30,3 @@ define([ return JQEntry; }); - diff --git a/static/js/collections/mdndomobjs.js b/static/js/collections/mdndomobjs.js index ac52d3e..d7c9987 100644 --- a/static/js/collections/mdndomobjs.js +++ b/static/js/collections/mdndomobjs.js @@ -2,18 +2,17 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/mdndomobj' -], function($, _, Backbone, MDNDomObj) { + 'models/sectionscrape' +], function($, _, Backbone, SectionScrape) { var MDNDomObj = Backbone.Collection.extend({ - url: '/mdndomobj', - model: MDNDomObj, + url: '/data/dom-mdn.json', + model: SectionScrape, comparator: function(model) { - return model.get('fullTitle'); + return model.get('title'); }, }); return MDNDomObj; }); - diff --git a/static/js/collections/mdnhtmlelements.js b/static/js/collections/mdnhtmlelements.js index 0e9c85c..c8b1bd2 100644 --- a/static/js/collections/mdnhtmlelements.js +++ b/static/js/collections/mdnhtmlelements.js @@ -2,16 +2,16 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/mdnhtmlelement' -], function($, _, Backbone, MDNHtmlElement) { + 'models/sectionscrape' +], function($, _, Backbone, SectionScrape) { // Handle the following cases: // var htmlElementPattern = new RegExp("^(\\W*)(.+)$"); var MDNHtmlElemnts = Backbone.Collection.extend({ - url: '/mdnhtmlelement', - model: MDNHtmlElement, + url: '/data/html-mdn.json', + model: SectionScrape, comparator: function(model) { var title = model.get('title'); diff --git a/static/js/collections/mdnjsobjs.js b/static/js/collections/mdnjsobjs.js index b4733e6..c43daf3 100644 --- a/static/js/collections/mdnjsobjs.js +++ b/static/js/collections/mdnjsobjs.js @@ -2,18 +2,18 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/mdnjsobj' -], function($, _, Backbone, MDNJsObj) { + 'models/sectionscrape' +], function($, _, Backbone, SectionScrape) { var MDNJsObj = Backbone.Collection.extend({ - url: '/mdnjsobj', - model: MDNJsObj, + url: '/data/js-mdn.json', + model: SectionScrape, comparator: function(model) { - return model.get('fullTitle'); + return model.get('title'); }, + }); return MDNJsObj; }); - diff --git a/static/js/collections/mozdevcssprops.js b/static/js/collections/mozdevcssprops.js index 52347ee..9bcd13c 100644 --- a/static/js/collections/mozdevcssprops.js +++ b/static/js/collections/mozdevcssprops.js @@ -2,8 +2,8 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/mozdevcssprop' -], function($, _, Backbone, MozDevCSSProp) { + 'models/sectionscrape' +], function($, _, Backbone, SectionScrape) { // Handle the following cases: // -xyz @@ -15,17 +15,17 @@ define([ var cssPropsPattern = new RegExp("^(\\W*)(.+)$"); var MozDevCSSProps = Backbone.Collection.extend({ - url: '/mozdevcssprop', - model: MozDevCSSProp, + url: '/data/css-mdn.json', + model: SectionScrape, - comparator: function(model) { - var title = model.get('title'); - var results = cssPropsPattern.exec(title); - var prefix = results[1]; - var name = results[2].toLowerCase(); + comparator: function(model) { + var title = model.get('title'); + var results = cssPropsPattern.exec(title); + var prefix = results[1]; + var name = results[2].toLowerCase(); - return prefix ? ('1' + title) : ('0' + name); - }, + return prefix ? ('1' + title) : ('0' + name); + }, }); diff --git a/static/js/collections/phpexts.js b/static/js/collections/phpexts.js index 2c704e8..b054a59 100644 --- a/static/js/collections/phpexts.js +++ b/static/js/collections/phpexts.js @@ -2,18 +2,17 @@ define([ 'jQuery', 'Underscore', 'Backbone', - 'models/phpext' -], function($, _, Backbone, PhpExt) { + 'models/SectionScrape' +], function($, _, Backbone, SectionScrape) { var PhpExt = Backbone.Collection.extend({ - url: '/phpext', - model: PhpExt, + url: '/data/php.json', + model: SectionScrape, comparator: function(model) { - return model.get('fullTitle').toLowerCase(); + return model.get('title').toLowerCase(); }, }); return PhpExt; }); - diff --git a/static/js/models/jqentry.js b/static/js/models/jqentry.js deleted file mode 100644 index 51a3f65..0000000 --- a/static/js/models/jqentry.js +++ /dev/null @@ -1,42 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var JQEntry = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/jqentry', - defaults: { - 'title' : '', - 'fullTitle' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return JQEntry; -}); - diff --git a/static/js/models/mdndomobj.js b/static/js/models/mdndomobj.js deleted file mode 100644 index 0193c6d..0000000 --- a/static/js/models/mdndomobj.js +++ /dev/null @@ -1,43 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var MDNDomObj = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/mdndomobj', - defaults: { - 'title' : '', - 'fullTitle' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - this.set({ title: this.get('fullTitle') }, { silent: true }); // TODO: fix this hack - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return MDNDomObj; -}); - diff --git a/static/js/models/mdnhtmlelement.js b/static/js/models/mdnhtmlelement.js deleted file mode 100644 index 9f9c066..0000000 --- a/static/js/models/mdnhtmlelement.js +++ /dev/null @@ -1,41 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var MDNHtmlElement = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/mdnhtmlelement', - defaults: { - 'title' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return MDNHtmlElement; -}); - diff --git a/static/js/models/mdnjsobj.js b/static/js/models/mdnjsobj.js deleted file mode 100644 index e0bd2ae..0000000 --- a/static/js/models/mdnjsobj.js +++ /dev/null @@ -1,43 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var MDNJsObj = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/mdnjsobj', - defaults: { - 'title' : '', - 'fullTitle' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - this.set({ title: this.get('fullTitle') }, { silent: true }); // TODO: fix this hack - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return MDNJsObj; -}); - diff --git a/static/js/models/mozdevcssprop.js b/static/js/models/mozdevcssprop.js deleted file mode 100644 index 279a7d3..0000000 --- a/static/js/models/mozdevcssprop.js +++ /dev/null @@ -1,41 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var MozDevCSSProp = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/mozdevcssprop', - defaults: { - 'title' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return MozDevCSSProp; -}); - diff --git a/static/js/models/phpext.js b/static/js/models/phpext.js deleted file mode 100644 index 40a7821..0000000 --- a/static/js/models/phpext.js +++ /dev/null @@ -1,42 +0,0 @@ -define([ - 'jQuery', - 'Underscore', - 'Backbone' -], function($, _, Backbone) { - - var PHPExtension = Backbone.Model.extend({ - - idAttribute: '_id', - - urlRoot: '/phpext', - defaults: { - 'title' : '', - 'fullTitle' : '', - 'htmlEscapedTitle' : '', // Set in initialize - 'sectionNames' : [], - 'sectionHTMLs' : [], - - // display properties - 'tocVisible' : false, - 'mainVisible' : false, - }, - - initialize: function() { - var htmlEscapedTitle = $('
').html(this.get('title')).text(); - this.set({ - htmlEscapedTitle: htmlEscapedTitle, - lowerCaseTitle: htmlEscapedTitle.toLowerCase() - }, { - silent: true - }); - }, - - url: function() { - return this.urlRoot + '/' + this.get('name'); - }, - - }); - - return PHPExtension; -}); -