switch to hourly storage for stats.

- allows showing an hourly chart on the 'today' view - fixes timezone issues when in 'today' view #134 - increases size of stats tables by factor 24, but that should be less of an issue after dbcadcd
usefathom · Nov 13, 2018 · 2ca1e0f · 2ca1e0f
1 parent 21c0f97
commit 2ca1e0f
Show file tree

Hide file tree

Showing 27 changed files with 333 additions and 218 deletions.
diff --git a/assets/src/css/styles.css b/assets/src/css/styles.css
@@ -123,9 +123,6 @@ div.delete a { color: red; }
 	.box-pages { grid-column: 2; grid-row: 2 ; }
 	.box-referrers { grid-column: 3; grid-row: 2; }
 
-	/* since we hide chart for views with less than a day worth of data, move tables to row 1 */
-	.ltday .box-pages, .ltday .box-referrers{ grid-row: 1; }
-
 	.half { display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; align-items: center; }
 	.half div { text-align: right; }
 	.half div.submit { text-align: left; }
@@ -137,8 +134,6 @@ div.delete a { color: red; }
 }
 
 
-
-
 .login-page.flex-rapper { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; }
 .login-rapper { text-align: left; width: 320px; }
 .login-page label { position: relative; }

diff --git a/assets/src/js/components/Chart.js b/assets/src/js/components/Chart.js
@@ -8,34 +8,25 @@ import * as d3 from 'd3';
 import 'd3-transition';
 d3.tip = require('d3-tip');
 
-const formatDay = d3.timeFormat("%e"),
+const 
-    formatMonth = d3.timeFormat("%b"),
+  formatHour = d3.timeFormat("%H"),
-    formatMonthDay = d3.timeFormat("%b %e"),
+  formatDay = d3.timeFormat("%e"),
-    formatYear = d3.timeFormat("%Y");
+  formatMonth = d3.timeFormat("%b"),
+  formatMonthDay = d3.timeFormat("%b %e"),
+  formatYear = d3.timeFormat("%Y");
 
 const t = d3.transition().duration(600).ease(d3.easeQuadOut);
 
-// tooltip
-const tip = d3.tip().attr('class', 'd3-tip').html((d) => (`
-  <div class="tip-heading">${d.Date.toLocaleDateString()}</div>
-  <div class="tip-content">
-    <div class="tip-pageviews">
-      <div class="tip-number">${d.Pageviews}</div>
-      <div class="tip-metric">Pageviews</div>
-    </div>
-    <div class="tip-visitors">
-      <div class="tip-number">${d.Visitors}</div>
-      <div class="tip-metric">Visitors</div>
-    </div>
-  </div>
-`));
-
 function padZero(s) {
   return s < 10 ? "0" + s : s;
 }
 
 function timeFormatPicker(n) {
   return function(d, i) {
+    if( n === 24 ) {
+      return formatHour(d);
+    }
+
     if(d.getDate() === 1) {
       return d.getMonth() === 0 ? formatYear(d) : formatMonth(d) 
     } 
@@ -50,67 +41,88 @@ function timeFormatPicker(n) {
   }
 }
 
-function prepareData(startUnix, endUnix, data) {
-  // add timezone offset back in to get local start date
-  const timezoneOffset = (new Date()).getTimezoneOffset() * 60;
-  let startDate = new Date((startUnix + timezoneOffset) * 1000);
-  let endDate = new Date((endUnix + timezoneOffset) * 1000);
-  let datamap = [];
-  let newData = [];
-
-   // create keyed array for quick date access
-  let length = data.length;
-  let d, dateParts, date, key;
-  for(var i=0;i<length;i++) {
-    d = data[i];
-    // replace date with actual date object & store in datamap
-    dateParts = d.Date.split('T')[0].split('-');
-    date = new Date(dateParts[0], dateParts[1]-1, dateParts[2], 0, 0, 0)
-    key = date.getFullYear() + "-" + padZero(date.getMonth() + 1) + "-" + padZero(date.getDate());
-    d.Date = date;
-    datamap[key] = d;
-  }
-
-  // make sure we have values for each date
-  let currentDate = startDate;
-  while(currentDate < endDate) {
-    key = currentDate.getFullYear() + "-" + padZero(currentDate.getMonth() + 1) + "-" + padZero(currentDate.getDate());
-    data = datamap[key] ? datamap[key] : {
-        "Pageviews": 0,
-        "Visitors": 0,
-        "Date": new Date(currentDate),
-     };
-
-    newData.push(data);  
-    currentDate.setDate(currentDate.getDate() + 1);
-  }
-
- return newData;
-}
-
-
-
 class Chart extends Component {
   constructor(props) {
     super(props)
 
     this.state = {
       loading: false,
       data: [],
+      diffInDays: 1,
+      hoursPerTick: 24,
     }
   }
 
   componentWillReceiveProps(newProps, newState) {
     if(!this.paramsChanged(this.props, newProps)) {
       return;
     }
+
+    let daysDiff = Math.round((newProps.before-newProps.after)/24/60/60);
+    let stepHours = daysDiff > 1 ? 24 : 1;
+    this.setState({
+      diffInDays: daysDiff,
+      hoursPerTick: stepHours,
+    })
 
     this.fetchData(newProps)
   }
 
   paramsChanged(o, n) {
     return o.siteId != n.siteId || o.before != n.before || o.after != n.after;
   }
+
+  @bind
+  prepareData(data) {
+    let startDate = new Date(this.props.after * 1000);
+    let endDate = new Date(this.props.before * 1000);
+    let newData = [];
+
+    // instantiate JS Date objects
+    data = data.map((d) => {
+      d.Date = new Date(d.Date)
+      return d
+    })
+
+    // make sure we have values for each date (so 0 value for gaps)
+    let currentDate = startDate, nextDate, tick, offset = 0;
+    while(currentDate < endDate) {
+      tick = {
+          "Pageviews": 0,
+          "Visitors": 0,
+          "Date": new Date(currentDate),
+      };
+
+      nextDate = new Date(currentDate)
+      nextDate.setHours(nextDate.getHours() + this.state.hoursPerTick);
+
+      // grab data that falls between currentDate & nextDate
+      for(let i=data.length-offset-1; i>=0; i--) {
+        if( data[i].Date > nextDate) {
+          break;
+        }
+
+         // increment offset so subsequent dates can skip first X items in array
+         offset += 1;
+
+        // continue to next item in array if we're still below our target date
+        if( data[i].Date < currentDate) {
+          continue;
+        }
+
+        // add to tick data
+        tick.Pageviews += data[i].Pageviews;
+        tick.Visitors += data[i].Visitors;
+      }
+
+      newData.push(tick);  
+      currentDate = nextDate;
+    }
+
+  return newData;
+  }
+
+
 
   @bind
   prepareChart() {
@@ -130,19 +142,28 @@ class Chart extends Component {
 
     this.x = d3.scaleBand().range([0, this.innerWidth]).padding(0.1)
     this.y = d3.scaleLinear().range([this.innerHeight, 0])
-    this.ctx.call(tip)
+
+      // tooltip
+    this.tip = d3.tip().attr('class', 'd3-tip').html((d) => {
+      let title = this.state.diffInDays <= 1 ? d.Date.toLocaleString() : d.Date.toLocaleDateString();
+      return (`<div class="tip-heading">${title}</div>
+      <div class="tip-content">
+        <div class="tip-pageviews">
+          <div class="tip-number">${d.Pageviews}</div>
+          <div class="tip-metric">Pageviews</div>
+        </div>
+        <div class="tip-visitors">
+          <div class="tip-number">${d.Visitors}</div>
+          <div class="tip-metric">Visitors</div>
+        </div>
+      </div>`)});
+    this.ctx.call(this.tip)
   }
 
   @bind
   redrawChart() {
     let data = this.state.data;
 
-    // hide chart & bail if we're trying to show less than 1 day worth of data
-    this.base.parentNode.style.display = data.length <= 1 ? 'none' : '';
-    if(data.length <= 1) {
-      return;
-    }
-
     if( ! this.ctx ) {
       this.prepareChart()
     }
@@ -156,8 +177,8 @@ class Chart extends Component {
     let yAxis = d3.axisLeft().scale(y).ticks(3).tickSize(-innerWidth)
     let xAxis = d3.axisBottom().scale(x).tickFormat(timeFormatPicker(data.length))
 
-     // hide all "day" ticks if we're watching more than 100 days of data
+     // hide all "day" ticks if we're watching more than 31 days of data
-    if(data.length > 100) {
+    if(data.length > 31) {
       xAxis.tickValues(data.filter(d => d.Date.getDate() === 1).map(d => d.Date))
     }
 
@@ -214,7 +235,7 @@ class Chart extends Component {
       .attr('y', (d) => y(d.Visitors))   
 
     // add event listeners for tooltips
-    days.on('mouseover', tip.show).on('mouseout', tip.hide)   
+    days.on('mouseover', this.tip.show).on('mouseout', this.tip.hide)   
   }
 
   @bind
@@ -228,7 +249,7 @@ class Chart extends Component {
           return;
         }
 
-        let chartData = prepareData(props.after, props.before, d);
+        let chartData = this.prepareData(d);
         this.setState({ 
           loading: false,
           data: chartData,

diff --git a/assets/src/js/components/DatePicker.js b/assets/src/js/components/DatePicker.js
@@ -10,16 +10,6 @@ const padZero = function(n){return n<10? '0'+n:''+n;}
 
 function getNow() {
   let now = new Date()
-  let tzOffset = now.getTimezoneOffset()  * 60 * 1000;
-
-  // if we're ahead of UTC, stick to UTC's "now"
-  // this is ugly but a sad necessity for now because we store and aggregate statistics using UTC dates (without time data)
-  // For those ahead of UTC, "today" will always be empty if they're checking early on in their day
-  // see https://github.com/usefathom/fathom/issues/134
-  if (tzOffset < 0) {
-    now.setTime(now.getTime() + tzOffset )
-  }
-
   return now
 }
 
@@ -116,8 +106,8 @@ class DatePicker extends Component {
 
     // create unix timestamps from local date objects
     let before, after;
-    before = Math.round((+endDate) / 1000) - endDate.getTimezoneOffset() * 60;
+    before = Math.round((+endDate) / 1000);
-    after = Math.round((+startDate) / 1000) - startDate.getTimezoneOffset() * 60;
+    after = Math.round((+startDate) / 1000);
 
     this.setState({
       period: period || '',

diff --git a/pkg/aggregator/store.go b/pkg/aggregator/store.go
@@ -10,7 +10,7 @@ import (
 )
 
 func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
+	cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02T15"))
 	if stats, ok := r.Sites[cacheKey]; ok {
 		return stats, nil
 
@@ -35,7 +35,7 @@ func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*mod
 }
 
 func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
 	if stats, ok := r.Pages[cacheKey]; ok {
 		return stats, nil
 	}
@@ -71,7 +71,7 @@ func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostn
 }
 
 func (agg *Aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
 	if stats, ok := r.Referrers[cacheKey]; ok {
 		return stats, nil
 	}

diff --git a/pkg/api/routes.go b/pkg/api/routes.go
@@ -23,10 +23,6 @@ func (api *API) Routes() *mux.Router {
 
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site", api.Authorize(HandlerFunc(api.GetSiteStatsHandler))).Methods(http.MethodGet)
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site/groupby/day", api.Authorize(HandlerFunc(api.GetSiteStatsPerDayHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/pageviews", api.Authorize(HandlerFunc(api.GetSiteStatsPageviewsHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/visitors", api.Authorize(HandlerFunc(api.GetSiteStatsVisitorsHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/duration", api.Authorize(HandlerFunc(api.GetSiteStatsDurationHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/bounces", api.Authorize(HandlerFunc(api.GetSiteStatsBouncesHandler))).Methods(http.MethodGet)
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site/realtime", api.Authorize(HandlerFunc(api.GetSiteStatsRealtimeHandler))).Methods(http.MethodGet)
 
 	r.Handle("/api/sites/{id:[0-9]+}/stats/pages", api.Authorize(HandlerFunc(api.GetPageStatsHandler))).Methods(http.MethodGet)

diff --git a/pkg/api/site_stats.go b/pkg/api/site_stats.go
@@ -14,46 +14,6 @@ func (api *API) GetSiteStatsHandler(w http.ResponseWriter, r *http.Request) erro
 	return respond(w, http.StatusOK, envelope{Data: result})
 }
 
-// URL: /api/stats/site/pageviews
-func (api *API) GetSiteStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetTotalSiteViews(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/visitors
-func (api *API) GetSiteStatsVisitorsHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetTotalSiteVisitors(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/duration
-func (api *API) GetSiteStatsDurationHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetAverageSiteDuration(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/bounces
-func (api *API) GetSiteStatsBouncesHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetAverageSiteBounceRate(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
 // URL: /api/stats/site/realtime
 func (api *API) GetSiteStatsRealtimeHandler(w http.ResponseWriter, r *http.Request) error {
 	params := GetRequestParams(r)

diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go
@@ -30,11 +30,6 @@ type Datastore interface {
 	GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
 	SaveSiteStats(*models.SiteStats) error
 	GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
-	GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
-	GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
-	GetTotalSiteSessions(int64, time.Time, time.Time) (int64, error)
-	GetAverageSiteDuration(int64, time.Time, time.Time) (float64, error)
-	GetAverageSiteBounceRate(int64, time.Time, time.Time) (float64, error)
 	GetRealtimeVisitorCount(int64) (int64, error)
 
 	// pageviews

diff --git a/pkg/datastore/sqlstore/config.go b/pkg/datastore/sqlstore/config.go
@@ -55,14 +55,13 @@ func (c *Config) DSN() string {
 		mc.DBName = c.Name
 		mc.Params = map[string]string{
 			"parseTime": "true",
-			"loc":       "Local",
 		}
 		if c.SSLMode != "" {
 			mc.Params["tls"] = c.SSLMode
 		}
 		dsn = mc.FormatDSN()
 	case SQLITE:
-		dsn = c.Name + "?_loc=auto&_busy_timeout=5000"
+		dsn = c.Name + "?_busy_timeout=5000"
 	}
 
 	return dsn