From a530713baed53700a9629083831d4427f86040a0 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Apr 2015 19:05:36 -0400 Subject: [PATCH] adding a file I missed to include and fixes to cache and removed loopback hack to reduce confusion. --- app/components/app.jsx | 3 + lib/cache.js | 216 ---------------------------------------- server.js | 96 ++++++------------ server/models/github.js | 46 +++++++-- 4 files changed, 75 insertions(+), 286 deletions(-) delete mode 100644 lib/cache.js diff --git a/app/components/app.jsx b/app/components/app.jsx index 688b00a..d104704 100644 --- a/app/components/app.jsx +++ b/app/components/app.jsx @@ -3,6 +3,7 @@ var Router = require("react-router"); var { Route, RouteHandler, Link, DefaultRoute } = Router; var { AuthBlock } = require("./auth.jsx"); var Add = require("./add.jsx"); +var Issues = require("./issues.jsx"); var { Now, Next } = require("./heartbeats.jsx"); var Upcoming = require("./upcoming.jsx"); var Homepage = require("./homepage.jsx"); @@ -27,6 +28,7 @@ var App = React.createClass({
  •  
  • Add Project
    • +
  • Issues
  • This Heartbeat
  • Next Heartbeat @@ -79,6 +81,7 @@ var App = React.createClass({ var routes = ( + diff --git a/lib/cache.js b/lib/cache.js deleted file mode 100644 index b398bf5..0000000 --- a/lib/cache.js +++ /dev/null @@ -1,216 +0,0 @@ -/** - * A simple wrapper around Redis and Memcached for caching URL data, - * with a fallback in memory cache if neither of those is available. - * The user configures via environment variables: - * - * - REDIS_URL or REDISCLOUD_URL or REDISTOGO_URL: redis server IP. - * The REDIS*_URL should be a single IP or hostname, not a list. - * You can also provide a port and password. - * - * - MEMCACHED_URL: the Memcached server URL(s). This should be a - * single IP or hostname, or a comma-separated list. It can include - * the port, e.g., :. - * - * NOTE: use only one of the above, not both. If neither is present, - * the in memory cache is used by default. - * - * - CACHE_EXPIRE: the time in seconds to keep data in the cache. - * The default is 15 minutes. - * - * The module provides two methods: - * - * 1) write( url, data ) - * - * Use this to cache data about a given url. The data should be - * an Object. The cache will hold this data, keyed on the URL - * until CACHE_EXPIRE. - * - * 2) read( url, callback ) - * - * This will take a given URL and attempt to pull the URL data - * from cache, otherwise giving back null in the callback. - */ - -var url = require( 'url' ), - util = require( 'util' ), - cacheExpire = process.env.CACHE_EXPIRE || 60 * 15, // 15 mins - redisURL = process.env.REDIS_URL || - process.env.REDISCLOUD_URL || - process.env.REDISTOGO_URL, - memcachedURL = process.env.MEMCACHED_URL, - cacheWrapper; - -/** - * Turn a stringified Object back into an Object. - */ -function toObject( str ) { - try { - return JSON.parse( str ); - } catch ( err ) { - return null; - } -} - -/** - * Setup a Redis cache, wrapped in cacheWrapper. - */ -function setupRedisCache() { - var redis; - util.log( 'Using Redis cache with ' + redisURL ); - - try { - redisURL = url.parse( redisURL ); - - // Depending on the format of the host, we may not get a proper hostname (e.g., - // 'localhost' vs. 'http://localhost'. Assume localhost if missing. - redisURL.hostname = redisURL.hostname || 'localhost'; - redisURL.port = redisURL.port || 6379; - redis = require( 'redis' ).createClient( redisURL.port, redisURL.hostname ); - - // If there's an error, kill the cacheWrapper - redis.on( 'error', function ( err ) { - util.error( 'Redis Error: ' + err ); - cacheWrapper = null; - }); - - // Wait til we get a ready signal from the server to set the cacheWrapper - redis.on( 'ready', function( err ) { - cacheWrapper = { - write: function( url, data ) { - redis.setex( url , cacheExpire, data ); - }, - read: function( url, callback ) { - redis.get( url, function( err, res ) { - if ( err ) { - callback( { error: err } ); - return; - } - - // If we get values, return them, otherwise, send null - // to indicate that we don't know. - if ( res ) { - callback( null, toObject( res ) ); - } else { - callback( null, null ); - } - }); - } - }; - }); - - // If the connection drops on the other end, kill the cacheWrapper - redis.on( 'end', function() { - util.error( 'Redis Connection Closed.' ); - cacheWrapper = null; - }); - - if ( redisURL.auth ) { - redis.auth ( redisURL.auth.split( ':' )[ 1 ] ); - } - } catch ( ex ) { - util.error( 'Failed to load Redis:' + ex ); - } -} - -/** - * Setup a Memcached cache, wrapped in cacheWrapper - */ -function setupMemcachedCache() { - var memcached; - util.log( 'Using Memcached cache with ' + memcachedURL ); - - try { - // We can take a comma-separated list of IPs/domains. Unlike Redis, - // the memcache node module expects a :? vs. a full URL. - var urlList = memcachedURL.split( ',' ).map( function( host ) { - var hostElems = host.split( ':' ), - hostname = hostElems[ 0 ] || 'localhost', - port = hostElems[ 1 ] || 11211; - return hostname + ':' + port; - }); - - memcached = new ( require( 'mc' ) ).Client( urlList ); - memcached.connect( function() { - cacheWrapper = { - write: function( url, data ) { - memcached.set( url, data, { exptime: cacheExpire }, function(){} ); - }, - read: function( url, callback ) { - memcached.get( url, function( err, res ) { - if ( err ) { - if ( err.type === 'NOT_FOUND' ) { - // Nothing in cache for these keys, return null URL data. - callback( null, null ); - return; - } else { - util.log( 'Memcached Error: ' + util.inspect(err) ); - callback( { error: err } ); - return; - } - } - callback( null, toObject( res[ url ] ) ); - }); - } - }; - }); - } catch ( ex ) { - util.error( 'Failed to load Memcached:' + ex ); - } -} - -/** - * Setup in memory cache, if redis/memcache aren't used/available. - */ -function setupMemoryCache() { - var memoryCache = {}; - util.log( 'Using Memory Cache' ); - cacheWrapper = { - write: function( url, data ) { - memoryCache[ url ] = data; - - var expire = setTimeout( function() { - delete memoryCache[ url ]; - }, cacheExpire * 1000 ); - expire.unref(); - }, - read: function( url, callback ) { - callback( null, toObject( memoryCache[ url ] ) ); - } - }; -} - -/** - * Check for config info for Redis and Memcached, use one or the other - */ -if ( redisURL ) { - setupRedisCache(); -} else if ( memcachedURL ) { - setupMemcachedCache(); -} else { - setupMemoryCache(); -} - -exports.write = function( url, data ) { - if ( cacheWrapper ) { - // Flatten our data into a string for storage - data = JSON.stringify( data ); - cacheWrapper.write( url, data ); - } -}; - -exports.read = function( url, callback ) { - if ( !cacheWrapper || !url ) { - callback( 'Error' ); - return; - } - - cacheWrapper.read( url, function( err, response ) { - // If we get an error back, or a null cache object (not found), bail - if ( err || !response ) { - callback( err, response ); - return; - } - callback( null, response ); - }); -}; - diff --git a/server.js b/server.js index 1317bec..a9b662a 100644 --- a/server.js +++ b/server.js @@ -21,7 +21,6 @@ var expressValidator = require('express-validator'); var issueParser = require('./server/issueparser.js'); var processHook = issueParser.processHook; var request = require( "request" ); -var cache = require( "./lib/cache" ); /** * Import API keys from environment @@ -31,8 +30,9 @@ var secrets = require('./server/config/secrets'); /** * Github handlers */ +var githubCacheAge = 60 * 60 * 1000; // every hour var Github = require('./server/models/github'); -var github = new Github(secrets.github); +var github = new Github(secrets.github, githubCacheAge); /** * Create Express server. @@ -48,6 +48,7 @@ var routes = require( "./routes" )(); * Express configuration. */ app.set('port', process.env.PORT || 8080); +app.set('host', process.env.HOST || "http://localhost"); app.set('github_org', 'MozillaFoundation'); app.set('github_repo', 'plan'); @@ -193,46 +194,19 @@ app.get('/issues', function(req, res) { res.sendFile(path.join(__dirname, './app/public/index.html')); }); -// Cache check middleware: if the URL is in cache, use that. -function checkCache( req, res, next ) { - if ( checkCache.overrides[ req.url ] ) { - delete checkCache.overrides[ req.url ]; - next(); - return; - } - cache.read( req.url, function( err, data ) { - if ( err || !data ) { - next( err ); - return; - } - res.json( data ); - }); -} -checkCache.overrides = {}; - -var mozillaRepos = "id.webmaker.org webmaker-curriculum snippets teach.webmaker.org goggles.webmaker.org webmaker-tests sawmill login.webmaker.org openbadges-badgekit webmaker-app api.webmaker.org popcorn.webmaker.org webmaker-mediasync webmaker.org webmaker-app-cordova webmaker-metrics nimble mozilla-opennews teach-api mozillafestival.org call-congress-net-neutrality thimble.webmaker.org advocacy.mozilla.org privacybadges webmaker-profile-2 call-congress build.webmaker.org webmaker-landing-pages webliteracymap events.webmaker.org badgekit-api openbadges-specification make-valet webmaker-auth webmaker-events-service webmaker-language-picker MakeAPI blog.webmaker.org webmaker-login-ux webmaker-desktop webmaker-app-publisher badges.mozilla.org lumberyard webmaker-download-locales webmaker-addons bsd-forms-and-wrappers popcorn-js hivelearningnetworks.org webmaker-firehose makeapi-client makerstrap webmaker-app-bot webmaker-screenshot react-i18n webmaker-kits-builder webmaker-app-guide".split(" "); -var orgs = ["MozillaFoundation", "MozillaScience"]; - -app.get( "/api/github/mozilla-repo-names", checkCache, function(req, res) { +app.get( "/api/github/mozilla-repo-names", function(req, res) { // Get Foundation repos then merge them with a static list of mozilla repos. - github.getRepos(orgs, function(err, results) { + github.getMozillaRepos(function(err, results) { if (err) { // Not sure what to do with errors, yet. console.log(err); } else { - var repoNames = []; - results.forEach(function(repo) { - repoNames.push(repo.full_name); - }); - // Merge with static list of mozilla repos. - res.json( repoNames.concat(mozillaRepos.map(function(item) { - return "mozilla/" + item; - }))); + res.json(results); } }); }); -app.get( "/api/github/foundation-users", checkCache, function(req, res) { - github.getUsersForOrgs(orgs, function(err, results) { +app.get( "/api/github/foundation-users", function(req, res) { + github.getFoundationUsers(function(err, results) { if (err) { // Not sure what to do with errors, yet. console.log(err); @@ -241,13 +215,13 @@ app.get( "/api/github/foundation-users", checkCache, function(req, res) { } }); }); -app.get( "/api/github/mozilla-labels", checkCache, function(req, res) { - var url = "http://127.0.0.1:" + app.get('port') + "/api/github/mozilla-repo-names"; - request(url, function (error, response, body) { - if (error) { - console.log(error); - } else if (!error && response.statusCode == 200) { - var repos = JSON.parse(body); +app.get( "/api/github/mozilla-labels", function(req, res) { + + github.getMozillaRepos(function(err, repos) { + if (err) { + // Not sure what to do with errors, yet. + console.log(err); + } else { github.getLabelsForRepos(repos, function(err, results) { if (err) { // Not sure what to do with errors, yet. @@ -259,13 +233,13 @@ app.get( "/api/github/mozilla-labels", checkCache, function(req, res) { } }); }); -app.get( "/api/github/mozilla-milestones", checkCache, function(req, res) { - var url = "http://127.0.0.1:" + app.get('port') + "/api/github/mozilla-repo-names"; - request(url, function (error, response, body) { - if (error) { - console.log(error); - } else if (!error && response.statusCode == 200) { - var repos = JSON.parse(body); +app.get( "/api/github/mozilla-milestones", function(req, res) { + + github.getMozillaRepos(function(err, repos) { + if (err) { + // Not sure what to do with errors, yet. + console.log(err); + } else { github.getMilestonesForRepos(repos, function(err, results) { if (err) { // Not sure what to do with errors, yet. @@ -277,45 +251,41 @@ app.get( "/api/github/mozilla-milestones", checkCache, function(req, res) { } }); }); -// To increase client-side performance, we prime the cache with data we'll need. -// Each resource (route URL) can specify a unique frequency for updates. If -// none is given, the cache expiration time is used. + function primeCache( urlPrefix ) { - // { url: "url-for-route", frequency: update-period-in-ms } [ { url: "/api/github/mozilla-repo-names" }, { url: "/api/github/foundation-users" }, { url: "/api/github/mozilla-labels" }, { url: "/api/github/mozilla-milestones" } - ].forEach( function( resource ) { - var url = resource.url, - frequency = resource.frequency || 60 * 60 * 1000; // Default: every hour + ].forEach( function( resource ) { + var url = resource.url; function updateResource() { - checkCache.overrides[ url ] = true; request.get( urlPrefix + url, function( err, resp, body ) { if ( err ) { return console.log( "Error updating cache entry for %s: %s", url, err ); } - cache.write( url, JSON.parse(body) ); }); } // Setup a timer to do this update, and also do one now updateResource(); - setInterval( updateResource, frequency ).unref(); - }); + setInterval( updateResource, githubCacheAge ).unref(); + }); } -primeCache("http://127.0.0.1:" + app.get('port')); +// Cache a few urls so the user always uses cache and never needs to wait. +primeCache(app.get("host") + ":" + app.get('port')); -/** - * Webhook handler (from github) - */ github.githubRequest({query:"rate_limit"}, function(err, data) { console.log("Github API requests left: " + data.rate.remaining); }); +/** + * Webhook handler (from github) + */ + /** * 500 Error Handler. */ diff --git a/server/models/github.js b/server/models/github.js index 3b78a74..21de303 100644 --- a/server/models/github.js +++ b/server/models/github.js @@ -12,7 +12,7 @@ var request = require('request'); /** * Constructor */ -function Github(githubSecrets) { +function Github(githubSecrets, cacheAge) { var _this = this; _this.client = githubSecrets.client; @@ -21,8 +21,8 @@ function Github(githubSecrets) { _this.host = 'https://api.github.com'; _this.repo = '/repos/MozillaFoundation/plan'; _this.cache = lru({ - max: 100, - maxAge: 1000 * 60 * 5 + max: 500, + maxAge: cacheAge || 60 * 60 * 1000 // default: every hour }); /** @@ -92,11 +92,15 @@ Github.prototype.githubRequest = function(options, callback) { var accessToken = this.token; var url = "https://api.github.com/" + options.query + "?access_token=" + accessToken + "&page="; var collection = []; + var copy = this.cache.get(url); + if (typeof copy !== 'undefined') { + return callback(null, copy); + } // Fetch deals with multiple pages. // The data this code deals with is small enough, // so just return all pages worth of data. - function fetch(page) { + var fetch = function(page) { request({ url: url + page, headers: { @@ -110,16 +114,18 @@ Github.prototype.githubRequest = function(options, callback) { fetch(++page); } else if (collection.length) { // Looks like we're done. + this.cache.set(url, collection); callback(error, collection); } else if (data.message) { // Likely an error. callback(data); } else { // Likely dealing with non array data. We can stop. + this.cache.set(url, data); callback(error, data); } - }); - } + }.bind(this)); + }.bind(this); fetch(0); }; @@ -197,6 +203,9 @@ Github.prototype.postIssueWithToken = function(token, body, callback) { }); }; +var mozillaRepos = "id.webmaker.org webmaker-curriculum snippets teach.webmaker.org goggles.webmaker.org webmaker-tests sawmill login.webmaker.org openbadges-badgekit webmaker-app api.webmaker.org popcorn.webmaker.org webmaker-mediasync webmaker.org webmaker-app-cordova webmaker-metrics nimble mozilla-opennews teach-api mozillafestival.org call-congress-net-neutrality thimble.webmaker.org advocacy.mozilla.org privacybadges webmaker-profile-2 call-congress build.webmaker.org webmaker-landing-pages webliteracymap events.webmaker.org badgekit-api openbadges-specification make-valet webmaker-auth webmaker-events-service webmaker-language-picker MakeAPI blog.webmaker.org webmaker-login-ux webmaker-desktop webmaker-app-publisher badges.mozilla.org lumberyard webmaker-download-locales webmaker-addons bsd-forms-and-wrappers popcorn-js hivelearningnetworks.org webmaker-firehose makeapi-client makerstrap webmaker-app-bot webmaker-screenshot react-i18n webmaker-kits-builder webmaker-app-guide".split(" "); +var foundationOrgs = ["MozillaFoundation", "MozillaScience"]; + /** * Returns an array of milestones from the "plan" repo. * @@ -207,7 +216,7 @@ Github.prototype.getMilestones = function(callback) { this.githubJSON(this.repo + '/milestones', callback); }; -Github.prototype.getRepos = function(orgs, callback) { +Github.prototype.getReposFromOrgs = function(orgs, callback) { async.concat(orgs, function(item, callback) { this.githubRequest({ query: "orgs/" + item + "/repos" @@ -217,6 +226,23 @@ Github.prototype.getRepos = function(orgs, callback) { }); }; +Github.prototype.getMozillaRepos = function(callback) { + this.getReposFromOrgs(foundationOrgs, function(err, results) { + if (err) { + // Not sure what to do with errors, yet. + callback(err); + } else { + var repoNames = []; + results.forEach(function(repo) { + repoNames.push(repo.full_name); + }); + // Merge with static list of mozilla repos. + callback(err, repoNames.concat(mozillaRepos.map(function(item) { + return "mozilla/" + item; + }))); + } + }); +}; Github.prototype.getUsersForOrgs = function(orgs, callback) { async.concat(orgs, function(item, callback) { this.githubRequest({ @@ -238,6 +264,12 @@ Github.prototype.getUsersForOrgs = function(orgs, callback) { }); }; +Github.prototype.getFoundationUsers = function(callback) { + this.getUsersForOrgs(foundationOrgs, function(err, results) { + callback(err, results); + }); +}; + Github.prototype.getMilestonesForRepos = function(repos, callback) { async.concat(repos, function(item, callback) { var orgName = item.repo;