Permalink
Browse files

Support getting more than 50 YouTube entries

YouTube API at the moment only allows `max-results` of maximum 50
but there are cases we want to get more than that.

The end-user can specify `max-results` to the number of entries
wanted in the `params`, and this will parse into multiple requests
using the `start-index` param.

Added `async` as dependency so that all the requests can be done
in parallel.

Bump version up.
  • Loading branch information...
1 parent 813c125 commit a31a47c3560d8cc6c20f85b6d7c2197834ee81a1 Son Tran-Nguyen committed Jun 28, 2013
Showing with 55 additions and 31 deletions.
  1. +52 −29 api.js
  2. +3 −2 package.json
View
81 api.js
@@ -1,39 +1,62 @@
var spashttp = require("spas-http"),
- _ = require("underscore")._;
+ _ = require("underscore")._,
+ async = require("async");
+
+var getVideoDetails = function (credentials) {
+ // If this bundle is using oauth2, add in the access token
+ var tokenString = _.isObject(credentials) && _.has(credentials, 'access_token') ?
+ "&access_token=" + credentials.access_token :
+ '';
+ return function (obj, cb) {
+ spashttp.request({url: "http://gdata.youtube.com/feeds/api/videos/" + obj.media$group.yt$videoid.$t + '?v=2&alt=json' + tokenString }, credentials, function( err, video ) {
+ if(video && _.has(video, 'entry')) {
+ obj.media$group.media$keywords.$t = video.entry.media$group.media$keywords.$t;
+ obj.category = video.entry.category;
+ }
+
+ cb(err);
+ });
+ }
+};
exports["custom"] = {
videosWithKeywords: function(params, credentials, cb) {
-
params.url = "http://gdata.youtube.com/feeds/api/videos?v=2";
-
- spashttp.request(params, credentials, function( err, videos ) {
-
- if (_.has(videos, 'feed')) {
-
- var n = videos.feed.entry.length;
-
- // If this bundle is using oauth2, add in the access token
- var tokenString = _.isObject(credentials) && _.has(credentials, 'access_token') ?
- "&access_token=" + credentials.access_token :
- '';
-
- _.each(videos.feed.entry, function( obj, key) {
- spashttp.request({url: "http://gdata.youtube.com/feeds/api/videos/" + obj.media$group.yt$videoid.$t + '?v=2&alt=json' + tokenString }, credentials, function( err, video ) {
- n=n-1;
- if(video && _.has(video, 'entry')) {
- videos.feed.entry[key].media$group.media$keywords.$t = video.entry.media$group.media$keywords.$t;
- videos.feed.entry[key].category = video.entry.category;
- }
- if (n === 0) {
- cb( null, videos );
- }
+ // Ensure we have a number to perform calculation.
+ var maxResults = parseInt(params['max-results']) || 50;
+ if (maxResults > 50) params['max-results'] = "50";
+ var pages = Math.floor((maxResults-1)/50) + 1;
+ var startIndices = [];
+ // Prep an array for starting indices to use with `async.concat`
+ for (var i = 0; i < pages; i++) { startIndices[i] = i*50 + 1; };
+
+ // In order to concat the video entries only
+ var data;
+
+ async.concat(startIndices, function (startIndex, callback) {
+ var shadowed = _.clone(params);
+ shadowed['start-index'] = startIndex+'';
+
+ spashttp.request(shadowed, credentials, function ( err, videos ) {
+ if (_.has(videos, 'feed')) {
+ // Save meta data outside.
+ if (!data) data = videos;
+
+ async.each(videos.feed.entry, getVideoDetails(credentials), function (err) {
+ callback(err, videos.feed.entry);
});
- });
+
+ } else {
+ callback( err, videos );
+ }
- } else {
- cb( err, videos );
- }
-
+ });
+ }, function(err, results) {
+ // After concatenation is done, save the entries back and return.
+ data.feed.entry = results;
jlafitte
jlafitte Sep 19, 2013 Contributor

I was trying to get the alt=jsonc feed working with this and it fails on this. Would be interested in how to get this working.

dtex
dtex Sep 19, 2013 Owner

Can you share your bundle object so we can try it out?

jlafitte
jlafitte Sep 19, 2013 Contributor

I talked to Son about it a little last night and I found where a google developer said that the json-c format was not being enhanced and would not be available in v3 so it might be a better idea to skip json-c support. We should probably consider cleaning up the output though, all the $ keys should be easier to read. Single pair objects (categories for example) would be better served as string arrays.

dtex
dtex Sep 19, 2013 Owner

Yes, there keys are obtuse and heard to read, but cleanup up the output should be handled by the specific case, not by spas-youtube. Right now we just wrap youtube's json response in our own object without changing the names or structures. This is the way to go because if someone wants to use existing code, they just call their existing callback from the spas callback. That sounds confusing... here, this explains it better: http://dtex.github.io/spas/tips.html

jlafitte
jlafitte Sep 19, 2013 Contributor

I understand.

dtex
dtex Sep 19, 2013 Owner

We could perhaps add an option to convert output to match v2 or v3 (disabled by default). That would be cool, but then I would think the bundle should just be updated to use the new API

+ cb(err, data);
});
+
+
}
};
View
@@ -1,7 +1,7 @@
{
"name": "spas-youtube",
"description": "YouTube API helpers for SPAS",
- "version": "0.1.3",
+ "version": "0.1.4",
"author": {
"name": "Donovan Buck",
"email": "donovan@donovan.bz",
@@ -21,7 +21,8 @@
],
"dependencies": {
"request": "2.x.x",
- "underscore": "1.x.x"
+ "underscore": "1.x.x",
+ "async": "0.2.5"
},
"main": "./api.js",
"engines": {

0 comments on commit a31a47c

Please sign in to comment.