feat(stats): update stats from beginning

src/database/recommendations.js

@@ -17,8 +17,13 @@ const transformRowToObject = function (row) {
     return
   }
 
-  row.created_date = new Date(row.created_date)
-  row.updated_date = new Date(row.updated_date)
+  if (row.created_date) {
+    row.created_date = new Date(row.created_date)
+  }
+
+  if (row.updated_date) {
+    row.updated_date = new Date(row.updated_date)
+  }
 
   if (row.liked_date) {
     row.liked_date = new Date(row.liked_date)
@@ -127,6 +132,30 @@ const buildWhereClause = (keys, values) => {
     .replace(/date\(\? AND/, 'date(?,')
 }
 
+const buildSelectClause = (unique, select) => {
+  if (unique && _.includes(select, 'liked_date')) {
+    const index = _.indexOf(select, 'liked_date')
+    select[ index ] = `strftime('%Y-%m-%d',${select[ index ]}) AS liked_date`
+  }
+
+  if (unique && _.includes(select, 'matched_date')) {
+    const index = _.indexOf(select, 'matched_date')
+    select[ index ] = `strftime('%Y-%m-%d',${select[ index ]}) AS matched_date`
+  }
+
+  if (unique && _.includes(select, 'trained_date')) {
+    const index = _.indexOf(select, 'trained_date')
+    select[ index ] = `strftime('%Y-%m-%d',${select[ index ]}) AS trained_date`
+  }
+
+  if (unique && _.includes(select, 'last_checked_out_date')) {
+    const index = _.indexOf(select, 'last_checked_out_date')
+    select[ index ] = `strftime('%Y-%m-%d',${select[ index ]}) AS last_checked_out_date`
+  }
+
+  return select
+}
+
 class Recommendations {
   save (channel, channelId, data) {
     const _data = transformObjectToRow(_.clone(data))
@@ -162,12 +191,13 @@ class Recommendations {
       .then(() => this.findByChannelAndChannelId(channel, channelId))
   }
 
-  findAll (page = 1, limit = 25, criteria, sort = 'last_checked_out_date') {
+  findAll (page = 1, limit = 25, criteria = {}, select = [ '*' ], sort = 'last_checked_out_date', unique = false) {
     const offset = (page - 1) * limit
 
     let queryResults
     let queryTotalCount
     let params = []
+    let _select = buildSelectClause(unique, select)
     if (!_.isEmpty(criteria)) {
       const _criteria = transformObjectToRow(_.clone(criteria))
 
@@ -176,12 +206,12 @@ class Recommendations {
 
       const where = buildWhereClause(keys, values)
 
-      queryResults = `SELECT * FROM recommendations WHERE ${where} ORDER BY ${sort} DESC LIMIT ${limit} OFFSET ${offset}`
-      queryTotalCount = `SELECT COUNT(*) as count FROM recommendations WHERE ${where}`
+      queryResults = `SELECT ${unique ? 'DISTINCT' : ''} ${_select} FROM recommendations WHERE ${where} ORDER BY ${sort} DESC LIMIT ${limit} OFFSET ${offset}`
+      queryTotalCount = `SELECT ${unique ? 'DISTINCT' : ''} COUNT(*) as count FROM recommendations WHERE ${where}`
       params = params.concat(values)
     } else {
-      queryResults = `SELECT * FROM recommendations ORDER BY ${sort} DESC LIMIT ${limit} OFFSET ${offset}`
-      queryTotalCount = 'SELECT COUNT(*) as count FROM recommendations'
+      queryResults = `SELECT ${unique ? 'DISTINCT' : ''} ${_select} FROM recommendations ORDER BY ${sort} DESC LIMIT ${limit} OFFSET ${offset}`
+      queryTotalCount = `SELECT ${unique ? 'DISTINCT' : ''} COUNT(*) as count FROM recommendations`
     }
 
     return Promise.props({

src/database/stats.js

@@ -74,7 +74,7 @@ class Stats {
     const offset = (page - 1) * limit
 
     return Promise.props({
-      results: queryAll.bind(this)(`SELECT * FROM stats ORDER BY date ASC LIMIT ${limit} OFFSET ${offset}`),
+      results: queryAll.bind(this)(`SELECT * FROM stats ORDER BY date DESC LIMIT ${limit} OFFSET ${offset}`),
       totalCount: SQLite.get('SELECT COUNT(*) as count FROM stats').then(({ count }) => count)
     })
   }

src/dates/dates.js

@@ -15,9 +15,10 @@ const Logger = require('modern-logger')
 const { Tinder } = require('../channels')
 const { NotAuthorizedError, OutOfLikesError } = require('../channels')
 const { Recommendation, AlreadyCheckedOutEarlierError } = require('./recommendation')
-const { SQLite, Recommendations, Channels, Stats } = require('../database')
+const { SQLite, Recommendations, Channels } = require('../database')
 
 const { Match } = require('./match')
+const Stats = require('./stats')
 
 const findAccount = (channel) => {
   return Channels.findByName(channel.name)
@@ -74,6 +75,12 @@ class Dates {
         .finally(() => Logger.info(`Finished finding dates in ${_.capitalize(channel.name)}`))
     }
 
+    const updateStats = () => {
+      return Logger.info('Started updating stats')
+        .then(() => Stats.update())
+        .finally(() => Logger.info('Finished updating stats'))
+    }
+
     return Channels.findAll()
       .mapSeries(({ name, is_enabled }) => {
         // eslint-disable-next-line camelcase
@@ -89,19 +96,19 @@ class Dates {
 
         return findByChannel.bind(this)(channel)
       })
-      .then(() => this.updateStats(new Date()))
+      .then(() => updateStats())
   }
 
   findByChannel (channel) {
     const checkRecommendations = function (channel) {
       return Logger.info(`Started checking recommendations from ${_.capitalize(channel.name)} `)
-      //.then(() => this.checkRecommendations(channel))
+      // .then(() => this.checkRecommendations(channel))
         .then(({ received = 0, skipped = 0, failed = 0 }) => Logger.info(`Finished checking recommendations from ${_.capitalize(channel.name)} (received = ${received}, skipped = ${skipped}, failed = ${failed})`))
     }
 
     const checkUpdates = function (channel) {
       return Logger.info(`Started checking updates from ${_.capitalize(channel.name)} `)
-        .then(() => this.checkUpdates(channel))
+      // .then(() => this.checkUpdates(channel))
         .then(({ matches = 0, messages = 0 }) => Logger.info(`Finished checking updates from ${_.capitalize(channel.name)} (matches = ${matches}, messages = ${messages})`))
     }
 
@@ -187,21 +194,6 @@ class Dates {
       })
       .catch((error) => Logger.error(error))
   }
-
-  updateStats (date) {
-    return Promise.props({
-      likes: Recommendations.findAll(1, 10000, { liked_date: date, like: 1 }),
-      passes: Recommendations.findAll(1, 10000, { last_checked_out_date: date, like: 0 }),
-      trains: Recommendations.findAll(1, 10000, { trained_date: date, train: 1 }),
-      matches: Recommendations.findAll(1, 10000, { matched_date: date, match: 1 })
-    })
-      .then(({ likes, passes, trains, matches }) => Stats.save(date, {
-        likes: likes.totalCount,
-        passes: passes.totalCount,
-        trains: trains.totalCount,
-        matches: matches.totalCount
-      }))
-  }
 }
 
 module.exports = new Dates()

src/dates/stats.js

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2017, Hugo Freire <hugo@exec.sh>.
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const _ = require('lodash')
+const Promise = require('bluebird')
+
+const Database = require('../database')
+
+const stats = [
+  { name: 'likes', metric: 'liked_date', criteria: { like: 1 } },
+  { name: 'passes', metric: 'last_checked_out_date', criteria: { like: 0 } },
+  { name: 'trains', metric: 'trained_date', criteria: { train: 1 } },
+  { name: 'matches', metric: 'matched_date', criteria: { match: 1 } }
+]
+
+class Stats {
+  update () {
+    return Database.Stats.findAll()
+      .then(({ results }) => {
+        if (_.isEmpty(results)) {
+          return this.updateFromStart()
+        }
+
+        const date = new Date()
+
+        return this.updateByDate(date)
+      })
+  }
+
+  updateFromStart () {
+    return Promise.mapSeries(stats, ({ name, metric, criteria }) => {
+      return Database.Recommendations.findAll(1, 1000, undefined, [ metric ], metric, true)
+        .then(({ results }) => results)
+        .mapSeries((result) => {
+          if (!result[ metric ]) {
+            return
+          }
+
+          return this.updateByStatsAndDate({ name, metric, criteria }, result[ metric ])
+        })
+    })
+  }
+
+  updateByDate (date) {
+    return Promise.mapSeries((stats), (stats) => this.updateByStatsAndDate(stats, date))
+  }
+
+  updateByStatsAndDate ({ name, metric, criteria }, date) {
+    const _criteria = _.merge({ [metric]: date }, criteria)
+
+    return Database.Recommendations.findAll(1, 10000, _criteria)
+      .then(({ totalCount }) => Database.Stats.save(date, { [name]: totalCount }))
+  }
+}
+
+module.exports = new Stats()

src/web/app/stats/stats.component.ts

@@ -60,17 +60,17 @@ export class StatsComponent {
   constructor (private statsService: StatsService) {}
 
   public ngOnInit () {
-    this.getPage(1, 31)
+    this.getPage(1, 15)
   }
 
-  getPage (page: number, limit: number = 31) {
+  getPage (page: number, limit: number = 25) {
     this.statsService.getAll(page, limit)
-      .subscribe(({ results, meta }) => {
-        this.lineChartLabels = _.map(results, 'date')
-        this.lineChartData[ 0 ].data = _.map(results, 'likes')
-        this.lineChartData[ 1 ].data = _.map(results, 'passes')
-        this.lineChartData[ 2 ].data = _.map(results, 'matches')
-        this.lineChartData[ 3 ].data = _.map(results, 'trains')
+      .subscribe(({ results }) => {
+        this.lineChartLabels = _.reverse(_.map(results, 'date'))
+        this.lineChartData[ 0 ].data = _.reverse(_.map(results, 'likes'))
+        this.lineChartData[ 1 ].data = _.reverse(_.map(results, 'passes'))
+        this.lineChartData[ 2 ].data = _.reverse(_.map(results, 'matches'))
+        this.lineChartData[ 3 ].data = _.reverse(_.map(results, 'trains'))
       })
   }
 }