metrics(metadata): Closes #1897 Track metadata coverage for sites

addon/ActivityStreams.js

@@ -86,6 +86,8 @@ function ActivityStreams(metadataStore, tabTracker, telemetrySender, options = {
   this._feeds = new FeedController({
     feeds,
     searchProvider: this._searchProvider,
+    metadataStore: this._metadataStore,
+    tabTracker: this._tabTracker,
     // TODO: move this into Feeds. Requires previewProvider/tabTracker to be independent
     getMetadata: (links, type) => {
       const event = this._tabTracker.generateEvent({source: type});

addon/Feeds/PlacesStatsFeed.js

@@ -1,4 +1,5 @@
 const {PlacesProvider} = require("addon/PlacesProvider");
+const {Cu} = require("chrome");
 const Feed = require("addon/lib/Feed");
 const am = require("common/action-manager");
 const {PlacesStatsUpdate} = am.actions;
@@ -7,6 +8,7 @@ const UPDATE_TIME = 24 * 60 * 60 * 1000; // 24 hours
 module.exports = class PlacesStatsFeed extends Feed {
   // Used by this.refresh
   getData() {
+    this.sendStatsPing();
     return Promise.all([
       PlacesProvider.links.getHistorySize(),
       PlacesProvider.links.getBookmarksSize()
@@ -15,6 +17,20 @@ module.exports = class PlacesStatsFeed extends Feed {
       PlacesStatsUpdate(historySize, bookmarksSize)
     ));
   }
+  sendStatsPing() {
+    this.options.metadataStore.asyncGetOldestInsert().then(timestamp => {
+      if (timestamp) {
+        Promise.all([
+          PlacesProvider.links.getHistorySizeSince(timestamp),
+          this.options.metadataStore.asyncCountAllItems()
+        ]).then(([placesCount, metadataCount]) => {
+          let event = this.options.tabTracker.generateEvent({source: "PLACES_STATS_FEED"});
+          this.options.tabTracker.handlePerformanceEvent(event, "countHistoryURLs", placesCount);
+          this.options.tabTracker.handlePerformanceEvent(event, "countMetadataURLs", metadataCount);
+        }).catch(e => Cu.reportError(e));
+      }
+    }).catch(e => Cu.reportError(e));
+  }
   onAction(state, action) {
     switch (action.type) {
       case am.type("APP_INIT"):

addon/MetadataStore.js

@@ -536,6 +536,38 @@ MetadataStore.prototype = {
     }
   }),
 
+  /**
+   * Find the oldest entry in the database
+   *
+   * Returns the timestamp of the oldest entry in the database
+   */
+  asyncGetOldestInsert: Task.async(function*() {
+    let timestamp = null;
+    try {
+      const entry = yield this.asyncExecuteQuery("SELECT min(created_at) FROM page_metadata");
+      if (entry && entry.length) {
+        timestamp = entry[0][0];
+      }
+    } catch (e) {
+      throw e;
+    }
+    return timestamp;
+  }),
+
+  /**
+   * Counts all the items in the database
+   *
+   * Returns a promise with the array of the retrieved metadata records
+   */
+  asyncCountAllItems: Task.async(function*() {
+    try {
+      const result = yield this.asyncExecuteQuery("SELECT count(*) FROM page_metadata");
+      return result[0][0];
+    } catch (e) {
+      throw e;
+    }
+  }),
+
   /**
   * Enables the data expiry job. The database connection needs to
   * be established prior to calling this function. Once it's triggered,

addon/PlacesProvider.js

@@ -760,6 +760,25 @@ Links.prototype = {
     return result[0][0];
   }),
 
+  /**
+   * Gets History size since a certain timestamp
+   *
+   * @param {String} timestamp in milliseconds
+   *
+   * @returns {Promise} Returns a promise with the count of moz_places records
+   *                    that have been entered since the timestamp provided
+   */
+  getHistorySizeSince: Task.async(function*(timestamp) {
+    let sqlQuery = `SELECT count(*)
+                    FROM moz_places WHERE id IN
+                    (SELECT DISTINCT place_id FROM moz_historyvisits
+                    WHERE datetime(visit_date / 1000 / 1000, 'unixepoch') >= :timestamp)
+                    AND hidden = 0 AND last_visit_date NOT NULL`;
+
+    let result = yield this.executePlacesQuery(sqlQuery, {params: {timestamp}});
+    return result[0][0];
+  }),
+
   /**
    * Gets Bookmarks count
    *

content-test/addon/Feeds/PlacesStatsFeed.test.js

@@ -1,7 +1,8 @@
 const PlacesProvider = {
   links: {
     getHistorySize: sinon.spy(() => Promise.resolve(42)),
-    getBookmarksSize: sinon.spy(() => Promise.resolve(1))
+    getBookmarksSize: sinon.spy(() => Promise.resolve(1)),
+    getHistorySizeSince: sinon.spy(() => Promise.resolve(0))
   }
 };
 const {PlacesStatsUpdate} = require("common/action-manager").actions;
@@ -13,7 +14,12 @@ describe("PlacesStatsFeed", () => {
   beforeEach(() => {
     PlacesProvider.links.getHistorySize.reset();
     PlacesProvider.links.getBookmarksSize.reset();
-    instance = new PlacesStatsFeed();
+    PlacesProvider.links.getHistorySizeSince.reset();
+    const MetadataStore = {
+      asyncGetOldestInsert: sinon.spy(() => Promise.resolve(10)),
+      asyncCountAllItems: sinon.spy(() => Promise.resolve(0))
+    };
+    instance = new PlacesStatsFeed({metadataStore: MetadataStore});
   });
   it("should create a PlacesStatsFeed", () => {
     assert.instanceOf(instance, PlacesStatsFeed);
@@ -29,7 +35,11 @@ describe("PlacesStatsFeed", () => {
     });
     it("should resolve with a PlacesStatsUpdate action", () => instance.getData().then(action => {
       assert.isObject(action);
-      assert.deepEqual(action, PlacesStatsUpdate(42, 1));
+      assert.deepEqual(action, PlacesStatsUpdate(42, 1, 0));
+    }));
+    it("should get the appropriate data from the metadata store", () => instance.getData().then(() => {
+      assert.calledOnce(instance.options.metadataStore.asyncGetOldestInsert);
+      assert.calledOnce(instance.options.metadataStore.asyncCountAllItems);
     }));
   });
 

test/lib/utils.js

@@ -99,7 +99,9 @@ function getTestActivityStream(options = {}) {
     asyncReset() {return Promise.resolve();},
     asyncClose() {return Promise.resolve();},
     asyncInsert() {return Promise.resolve();},
-    asyncGetMetadataByCacheKey() {return Promise.resolve([]);}
+    asyncGetMetadataByCacheKey() {return Promise.resolve([]);},
+    asyncGetOldestInsert() {return Promise.resolve([0]);},
+    asyncCountAllItems() {return Promise.resolve([0]);}
   };
   const mockShareProvider = {
     init() {},

test/test-MetadataStore.js

@@ -181,6 +181,52 @@ exports.test_async_insert_all = function*(assert) {
   assert.deepEqual(items[7], [3, 5]);
 };
 
+exports.test_get_oldest_insert = function*(assert) {
+  // insert metadata at 3 different times
+  for (let metadata of metadataFixture) {
+    yield gMetadataStore.asyncInsert([metadata]);
+  }
+  // get all the timestamps, pick an entry, set that timestamp to be an hour earlier
+  const metadataIdRows = yield gMetadataStore.asyncExecuteQuery("SELECT id FROM page_metadata LIMIT 1");
+  const metadataId = metadataIdRows[0][0];
+  const expectedOldestTimestamp = Math.floor((Date.now() - (60 * 60 * 1000)) / 1000);
+  yield gMetadataStore.asyncExecuteQuery(
+    `UPDATE page_metadata SET created_at=datetime(${expectedOldestTimestamp}, 'unixepoch', 'localtime') WHERE id=${metadataId}`);
+
+  // get the timestamp back out from the database the way it was inserted
+  // compare it directly against the timestamp from the new function
+  const expectedOldestDateRows = yield gMetadataStore.asyncExecuteQuery("SELECT min(created_at) FROM page_metadata");
+  const expectedOldestDate = expectedOldestDateRows[0][0];
+  const actualOldestDate = yield gMetadataStore.asyncGetOldestInsert();
+
+  assert.equal(typeof actualOldestDate, "string", `Expected ${actualOldestDate} to be a string`);
+  assert.deepEqual(expectedOldestDate, actualOldestDate, "Got the oldest timestamp");
+};
+
+exports.test_get_oldest_insert_returns_null = function*(assert) {
+  const metadataStore = new MetadataStore();
+  yield metadataStore.asyncConnect();
+  metadataStore.asyncExecuteQuery = function() {
+    return null;
+  };
+
+  const oldestEntry = yield metadataStore.asyncGetOldestInsert();
+  assert.deepEqual(oldestEntry, null, "Return a null entry if we didn't find any");
+
+  yield metadataStore.asyncTearDown();
+};
+
+exports.test_count_all_items = function*(assert) {
+  let itemsinDB = yield gMetadataStore.asyncCountAllItems();
+  assert.equal(itemsinDB, 0, "Prior to inserting we return 0 items");
+
+  yield gMetadataStore.asyncInsert(metadataFixture);
+
+  itemsinDB = yield gMetadataStore.asyncCountAllItems();
+  assert.equal(itemsinDB, 3, "Retrieved all items in the database");
+  assert.equal(typeof itemsinDB, "number", "Function returns a number");
+};
+
 exports.test_async_get_by_cache_key = function*(assert) {
   yield gMetadataStore.asyncInsert(metadataFixture);
 

test/test-PlacesProvider.js

@@ -35,6 +35,15 @@ function isVisitDateOK(timestampMS) {
   return Math.abs(Date.now() - timestampMS) < range;
 }
 
+// turns a timestamp from ISO format 2015-08-04T19:22:39.000Z into ISO format
+// 2015-08-04 19:22:39 so we can compare timestamps properly
+function formatISODate(timestamp) {
+  return new Date(timestamp).toISOString()
+                            .split("T")
+                            .join(" ")
+                            .split(".")[0];
+}
+
 exports.test_LinkChecker_securityCheck = function(assert) {
   let urls = [
     {url: "file://home/file/image.png", expected: false},
@@ -538,6 +547,28 @@ exports.test_Links_getHistorySize = function*(assert) {
   assert.equal(size, 1, "expected history size");
 };
 
+exports.test_Links_getHistorySizeSince = function*(assert) {
+  let provider = PlacesProvider.links;
+
+  let size = yield provider.getHistorySizeSince(null);
+  assert.equal(size, 0, "return 0 if there is no timestamp provided");
+
+  // add a visit
+  let testURI = NetUtil.newURI("http://mozilla.com");
+  yield PlacesTestUtils.addVisits(testURI);
+
+  // check that the history size updated with the visit
+  let timestamp = formatISODate(Date.now() - 10 * 60 * 1000);
+  size = yield provider.getHistorySizeSince(timestamp);
+  assert.equal(size, 1, "expected history size since the timestamp");
+  assert.equal(typeof size, "number", "function returns a number");
+
+  // add 10m and make sure we don't get that entry back
+  timestamp = formatISODate(Date.now() + 10 * 60 * 1000);
+  size = yield provider.getHistorySizeSince(timestamp);
+  assert.equal(size, 0, "do not return an entry");
+};
+
 exports.test_blocked_urls = function*(assert) {
   let provider = PlacesProvider.links;
   let {TRANSITION_TYPED} = PlacesUtils.history;