merge latest

src/github.com/getlantern/lantern/feed.go → src/github.com/getlantern/flashlight/feed/feed.go

@@ -1,4 +1,4 @@
-package lantern
+package feed
 
 import (
 	"compress/gzip"
@@ -7,27 +7,32 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/getlantern/eventual"
+	"github.com/getlantern/flashlight/geolookup"
 	"github.com/getlantern/flashlight/util"
+	"github.com/getlantern/golog"
 )
 
 const (
 	// the Feed endpoint where recent content is published to
 	// mostly just a compendium of RSS feeds
-	feedEndpoint = `https://feeds.getiantem.org/%s/feed.json`
+	feedEndpoint = "https://feeds.getiantem.org/%s/feed.json"
+	en           = "en_US"
 )
 
 var (
 	feed *Feed
-
 	// locales we have separate feeds available for
 	supportedLocales = map[string]bool{
 		"en_US": true,
 		"fa_IR": true,
 		"fa":    true,
+		"ms_MY": true,
 		"zh_CN": true,
 	}
+	log = golog.LoggerFor("feed")
 )
 
 // Feed contains the data we get back
@@ -63,13 +68,17 @@ type FeedItem struct {
 type FeedItems []*FeedItem
 
 type FeedProvider interface {
+	// AddSource: called for each new feed source
 	AddSource(string)
 }
 
 type FeedRetriever interface {
+	// AddFeed: used to add a new entry to a given feed
 	AddFeed(string, string, string, string)
 }
 
+// FeedByName checks the previously created feed for an
+// entry with the given source name
 func FeedByName(name string, retriever FeedRetriever) {
 	if feed != nil && feed.Items != nil {
 		if items, exists := feed.Items[name]; exists {
@@ -115,12 +124,12 @@ func GetFeed(locale string, allStr string, proxyAddr string,
 	if !supportedLocales[locale] {
 		// always default to English if we don't
 		// have a feed available in a specific locale
-		locale = "en_US"
+		locale = en
 	}
 
-	feedUrl := fmt.Sprintf(feedEndpoint, locale)
+	feedURL := GetFeedURL(locale)
 
-	if req, err = http.NewRequest("GET", feedUrl, nil); err != nil {
+	if req, err = http.NewRequest("GET", feedURL, nil); err != nil {
 		handleError(fmt.Errorf("Error fetching feed: %v", err))
 		return
 	}
@@ -220,3 +229,38 @@ func processFeed(allStr string, provider FeedProvider) {
 		}
 	}
 }
+
+// GetFeedURL returns the URL to use for looking up the feed by looking up
+// the users country before defaulting to the specified default locale if the
+// country can't be determined.
+func GetFeedURL(defaultLocale string) string {
+	locale := determineLocale(defaultLocale)
+	url := fmt.Sprintf(feedEndpoint, locale)
+	log.Debugf("Returning feed URL: %v", url)
+	return url
+}
+
+func determineLocale(defaultLocale string) string {
+	if defaultLocale == "" {
+		defaultLocale = en
+	}
+	// As of this writing the only countries we know of where we want a unique
+	// feed for the country that's different from the dominantly installed
+	// language are Iran and Malaysia. In both countries english is the most
+	// common language on people's machines. We can therefor optimize a little
+	// bit here and skip the country lookup if the locale is not en_US.
+	if !strings.EqualFold(en, defaultLocale) {
+		return defaultLocale
+	}
+	country := geolookup.GetCountry(time.Duration(10) * time.Second)
+	if country == "" {
+		// This means the country lookup failed, so just use whatever the default is.
+		log.Debug("Could not lookup country")
+		return defaultLocale
+	} else if strings.EqualFold("ir", country) {
+		return "fa_IR"
+	} else if strings.EqualFold("my", country) {
+		return "ms_MY"
+	}
+	return defaultLocale
+}

src/github.com/getlantern/lantern/feed_test.go → src/github.com/getlantern/flashlight/feed/feed_test.go

@@ -1,4 +1,4 @@
-package lantern
+package feed
 
 import (
 	"testing"

src/github.com/getlantern/flashlight/main/pac.go

@@ -162,6 +162,7 @@ func pacOff() {
 }
 
 func cyclePAC() {
+	log.Debug("Cycling the pac file")
 	// prevents Lantern from accidently leave pac on after exits
 	if atomic.LoadInt32(&isPacOn) == 1 {
 		// reapply so browser will fetch the PAC URL again

src/github.com/getlantern/flashlight/ui/ui.go

@@ -17,6 +17,7 @@ import (
 	"github.com/skratchdot/open-golang/open"
 
 	"github.com/getlantern/flashlight/client"
+	"github.com/getlantern/flashlight/feed"
 )
 
 const (
@@ -97,7 +98,18 @@ func Start(requestedAddr string, allowRemote bool, extUrl string) (string, error
 		}
 		resp.WriteHeader(http.StatusOK)
 	}
+
+	// We use the backend to detect the user's country and redirect the browser
+	// to the correct URL that will itself be proxied over Lantern.
+	feedHandler := func(resp http.ResponseWriter, req *http.Request) {
+		vals := req.URL.Query()
+		defaultLang := vals.Get("lang")
+		url := feed.GetFeedURL(defaultLang)
+		http.Redirect(resp, req, url, http.StatusFound)
+	}
+
 	r.Handle("/startup", http.HandlerFunc(handler))
+	r.Handle("/feed", http.HandlerFunc(feedHandler))
 	r.Handle("/", http.FileServer(fs))
 
 	server = &http.Server{

src/github.com/getlantern/lantern-ui/app/_css/news.css

@@ -172,7 +172,7 @@
   font-size: 10px;
 }
 
-#news p.feed-date {
+#news p.feed-meta {
   padding-top: 13px;
   font-size: 10px;
 }

src/github.com/getlantern/lantern-ui/app/angular-feed/scripts/directives.js

@@ -33,10 +33,50 @@ angular.module('feeds-directives', []).directive('feed', ['feedService', '$compi
         return feedEntry;
       }
 
-      function sanitizeEntries(entries) {
+      // Add/replace below fields to an entry:
+      // 1. the source field of an entry is the key of the feed, we need the feed title instead.
+      // 2. if the feed one entry belongs to needs to be excluded from All tab, apply to the entry itself.
+      function updateEntryFields(feedEntry, feeds) {
+          var source = feedEntry.source;
+          if (source) {
+            var feed = feeds[source];
+            feedEntry.excludeFromAll = feed.excludeFromAll;
+            if (feed && feed.title) {
+              feedEntry.source = feed.title;
+            }
+          }
+      }
+
+      function sanitizeEntries(entries, feeds) {
         for (var i = 0; i < entries.length; i++) {
           sanitizeFeedEntry(entries[i]);
+          updateEntryFields(entries[i], feeds);
+        }
+      }
+
+      // convert the feeds object to an array with the order specified by an array of keys.
+      function sort(feeds, order) {
+        var sorted = []
+        for (var key in order) {
+          var item = feeds[order[key]]
+          if (item) {
+            sorted.push(item)
+          } else {
+            console.error("feed " + item + " is not found in feeds!")
+          }
         }
+        return sorted
+      }
+
+      // feeds.entries is a list of indexes in allEntries, replace them with actual entries
+      function replaceWithRealEntries(feeds, allEntries) {
+        for (var i in feeds) {
+          var feedEntries = feeds[i].entries
+          for (var j in feedEntries) {
+            feedEntries[j] = allEntries[feedEntries[j]]
+          }
+        }
+        return feeds
       }
 
       var templateRendered = false;
@@ -48,9 +88,9 @@ angular.module('feeds-directives', []).directive('feed', ['feedService', '$compi
       }
 
       function render(feedsObj) {
-        sanitizeEntries(feedsObj.entries);
+        sanitizeEntries(feedsObj.entries, feedsObj.feeds);
         $scope.allEntries = feedsObj.entries;
-        $scope.allFeeds = feedsObj.feeds;
+        $scope.allFeeds = replaceWithRealEntries(sort(feedsObj.feeds, feedsObj.sorted_feeds), feedsObj.entries);
         if ($attrs.templateUrl) {
           $http.get($attrs.templateUrl, {cache: $templateCache}).success(function (templateHtml) {
             renderTemplate(templateHtml);
@@ -68,21 +108,21 @@ angular.module('feeds-directives', []).directive('feed', ['feedService', '$compi
           console.log("show feeds in cache");
           render(feedsObj);
           deferred.resolve(feedsObj);
-        } else {
-          feedService.getFeeds(url, $attrs.fallbackUrl, gaMgr).then(function (feedsObj) {
-            console.log("fresh copy of feeds loaded");
-            feedCache.set(url, feedsObj);
-            render(feedsObj);
-            deferred.resolve(feedsObj);
-          },function (error) {
-            console.error("fail to fetch feeds: " +  error);
-            if ($scope.onError) {
-              $scope.onError(error);
-            }
-            $scope.error = error;
-          });
         }
 
+        feedService.getFeeds(url, $attrs.fallbackUrl, gaMgr).then(function (feedsObj) {
+          console.log("fresh copy of feeds loaded");
+          feedCache.set(url, feedsObj);
+          render(feedsObj);
+          deferred.resolve(feedsObj);
+        },function (error) {
+          console.error("fail to fetch feeds: " +  error);
+          if ($scope.onError) {
+            $scope.onError(error);
+          }
+          $scope.error = error;
+        });
+
         deferred.promise.then(function(feedsObj) {
           if ($scope.onFeedsLoaded) {
             $scope.onFeedsLoaded();

src/github.com/getlantern/lantern-ui/app/angular-feed/scripts/services.js

@@ -37,7 +37,7 @@ angular.module('feeds-services', []).factory('feedService', ['$q', '$http', func
 }])
 
 .factory('feedCache', function () {
-  var CACHE_INTERVAL = 1000 * 60 * 20; // 20 minutes
+  var CACHE_INTERVAL = 1000 * 60 * 50 * 24; // 1 day
 
   function cacheTimes() {
     if ('CACHE_TIMES' in localStorage) {
@@ -48,20 +48,23 @@ angular.module('feeds-services', []).factory('feedService', ['$q', '$http', func
 
   function hasCache(name) {
     var CACHE_TIMES = cacheTimes();
-    var timeInCache = name in CACHE_TIMES && name in localStorage && new Date().getTime() - CACHE_TIMES[name];
-    return timeInCache < CACHE_INTERVAL;
+    return name in CACHE_TIMES && name in localStorage && new Date().getTime() - CACHE_TIMES[name] < CACHE_INTERVAL;
   }
 
   return {
     set: function (name, obj) {
-      localStorage.setItem(name, angular.toJson(obj));
+      var str = angular.toJson(obj);
+      var compressed = LZString.compressToUTF16(str);
+      localStorage.setItem(name, compressed);
       var CACHE_TIMES = cacheTimes();
       CACHE_TIMES[name] = new Date().getTime();
       localStorage.setItem('CACHE_TIMES', angular.toJson(CACHE_TIMES));
     },
     get: function (name) {
       if (hasCache(name)) {
-        return angular.fromJson(localStorage.getItem(name));
+        var compressed = localStorage.getItem(name);
+        var str = LZString.decompressFromUTF16(compressed);
+        return angular.fromJson(str);
       }
       return null;
     },

src/github.com/getlantern/lantern-ui/app/index.html

@@ -38,6 +38,7 @@
     <script src="lib/angular-ui-bootstrap-tpls.js"></script>
 
     <script src="lib/mailer.js"></script>
+    <script src="lib/lz-string.min.js"></script>
 
     <script src="bower_components/clipboard/dist/clipboard.min.js"></script>
     <script src="bower_components/ngclipboard/src/ngclipboard.js"></script>
@@ -47,9 +48,6 @@
 
     <!-- we host a copy of google's analytics.js locally -->
     <script src="lib/analytics.js"></script>
-    <!--<script src="bower_components/ui-bootstrap/src/tooltip/tooltip.js"></script>-->
-    <!-- XXX see "XXX" in js/vis.js before switching to more recent topojson -->
-    <script src="lib/topojson.js"></script>
     <!-- endbuild -->
 
     <!-- build:js2 js/app.js -->
@@ -65,7 +63,6 @@
     <script src="js/services.js"></script>
     <script src="js/controllers.js"></script>
     <script src="js/directives.js"></script>
-    <script src="js/vis.js"></script>
     <!-- endbuild -->
 
     <link rel="shortcut icon" href="img/favicon.ico">
@@ -77,8 +74,7 @@
   </body>
 <![endif]-->
 <!--[if (!IE)|(gte IE 10)]><!-->
-  <body ng-controller="RootCtrl" ng-cloak lang="{{ lang }}" dir="{{ langDirection }}"
-    ng-class="{showVis: model.showVis, getMode: inGetMode, giveMode: inGiveMode}">
+  <body ng-controller="RootCtrl" ng-cloak lang="{{ lang }}" dir="{{ langDirection }}">
     <div id="fb-root"></div>
     <div id="waiting" ng-show="!wsConnected">
     <div ng-show="!backendIsGone">

src/github.com/getlantern/lantern-ui/app/js/app.js

@@ -9,7 +9,6 @@ var app = angular.module('app', [
   'app.filters',
   'app.services',
   'app.directives',
-  'app.vis',
   'ngSanitize',
   'ngResource',
   'ngclipboard',

src/github.com/getlantern/lantern-ui/app/js/controllers.js

@@ -201,9 +201,10 @@ app.controller('NewsfeedCtrl', ['$scope', '$rootScope', '$translate', function($
     };
     var lang = $translate.use();
     lang = mapTable[lang] || lang;
-    return "https://feeds.getiantem.org/" + lang + "/feed.json";
-  };
-
+    var url = "/feed?lang="+lang;
+    console.log("Requesting feed from "+url);
+    return url;
+  }
 }]);
 
 app.controller('FeedTabCtrl', ['$scope', '$rootScope', '$translate', function($scope, $rootScope, $translate) {
@@ -241,7 +242,7 @@ app.controller('FeedCtrl', ['$scope', 'gaMgr', function($scope, gaMgr) {
     if (feed.meta && feed.meta.description) {
       return feed.meta.description;
     }
-    return feed.contentSnippet;
+    return feed.contentSnippetText;
   };
   $scope.trackFeed = function(name) {
     return gaMgr.trackFeed(name);