Skip to content
Permalink
Browse files

switch to hourly storage for stats.

- allows showing an hourly chart on the 'today' view
- fixes timezone issues when in 'today' view #134
- increases size of stats tables by factor 24, but that should be less of an issue after dbcadcd
  • Loading branch information...
dannyvankooten committed Nov 13, 2018
1 parent 21c0f97 commit 2ca1e0f36e049fd429eace2f33f5ecbd647bc10b
Showing with 333 additions and 218 deletions.
  1. +0 −5 assets/src/css/styles.css
  2. +91 −70 assets/src/js/components/Chart.js
  3. +2 −12 assets/src/js/components/DatePicker.js
  4. +3 −3 pkg/aggregator/store.go
  5. +0 −4 pkg/api/routes.go
  6. +0 −40 pkg/api/site_stats.go
  7. +0 −5 pkg/datastore/datastore.go
  8. +1 −2 pkg/datastore/sqlstore/config.go
  9. +19 −0 pkg/datastore/sqlstore/migrations/mysql/21_alter_page_stats_table.sql
  10. +17 −0 pkg/datastore/sqlstore/migrations/mysql/22_alter_site_stats_table.sql
  11. +19 −0 pkg/datastore/sqlstore/migrations/mysql/23_alter_referrer_stats_table.sql
  12. +9 −0 pkg/datastore/sqlstore/migrations/mysql/24_recreate_stat_table_indices.sql
  13. +20 −0 pkg/datastore/sqlstore/migrations/postgres/22_alter_page_stats_table.sql
  14. +19 −0 pkg/datastore/sqlstore/migrations/postgres/23_alter_referrer_stats_table.sql
  15. +17 −0 pkg/datastore/sqlstore/migrations/postgres/24_alter_site_stats_table.sql
  16. +11 −0 pkg/datastore/sqlstore/migrations/postgres/25_recreate_stat_table_indices.sql
  17. +19 −0 pkg/datastore/sqlstore/migrations/sqlite3/21_alter_page_stats_table.sql
  18. +17 −0 pkg/datastore/sqlstore/migrations/sqlite3/22_alter_site_stats_table.sql
  19. +19 −0 pkg/datastore/sqlstore/migrations/sqlite3/23_alter_referrer_stats_table.sql
  20. +11 −0 pkg/datastore/sqlstore/migrations/sqlite3/24_recreate_stat_table_indices.sql
  21. +11 −11 pkg/datastore/sqlstore/page_stats.go
  22. +11 −11 pkg/datastore/sqlstore/referrer_stats.go
  23. +12 −52 pkg/datastore/sqlstore/site_stats.go
  24. +2 −0 pkg/datastore/sqlstore/sqlstore.go
  25. +1 −1 pkg/models/page_stats.go
  26. +1 −1 pkg/models/referrer_stats.go
  27. +1 −1 pkg/models/site_stats.go
@@ -123,9 +123,6 @@ div.delete a { color: red; }
.box-pages { grid-column: 2; grid-row: 2 ; }
.box-referrers { grid-column: 3; grid-row: 2; }

/* since we hide chart for views with less than a day worth of data, move tables to row 1 */
.ltday .box-pages, .ltday .box-referrers{ grid-row: 1; }

.half { display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; align-items: center; }
.half div { text-align: right; }
.half div.submit { text-align: left; }
@@ -137,8 +134,6 @@ div.delete a { color: red; }
}




.login-page.flex-rapper { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; }
.login-rapper { text-align: left; width: 320px; }
.login-page label { position: relative; }
@@ -8,34 +8,25 @@ import * as d3 from 'd3';
import 'd3-transition';
d3.tip = require('d3-tip');

const formatDay = d3.timeFormat("%e"),
formatMonth = d3.timeFormat("%b"),
formatMonthDay = d3.timeFormat("%b %e"),
formatYear = d3.timeFormat("%Y");
const
formatHour = d3.timeFormat("%H"),
formatDay = d3.timeFormat("%e"),
formatMonth = d3.timeFormat("%b"),
formatMonthDay = d3.timeFormat("%b %e"),
formatYear = d3.timeFormat("%Y");

const t = d3.transition().duration(600).ease(d3.easeQuadOut);

// tooltip
const tip = d3.tip().attr('class', 'd3-tip').html((d) => (`
<div class="tip-heading">${d.Date.toLocaleDateString()}</div>
<div class="tip-content">
<div class="tip-pageviews">
<div class="tip-number">${d.Pageviews}</div>
<div class="tip-metric">Pageviews</div>
</div>
<div class="tip-visitors">
<div class="tip-number">${d.Visitors}</div>
<div class="tip-metric">Visitors</div>
</div>
</div>
`));

function padZero(s) {
return s < 10 ? "0" + s : s;
}

function timeFormatPicker(n) {
return function(d, i) {
if( n === 24 ) {
return formatHour(d);
}

if(d.getDate() === 1) {
return d.getMonth() === 0 ? formatYear(d) : formatMonth(d)
}
@@ -50,67 +41,88 @@ function timeFormatPicker(n) {
}
}

function prepareData(startUnix, endUnix, data) {
// add timezone offset back in to get local start date
const timezoneOffset = (new Date()).getTimezoneOffset() * 60;
let startDate = new Date((startUnix + timezoneOffset) * 1000);
let endDate = new Date((endUnix + timezoneOffset) * 1000);
let datamap = [];
let newData = [];

// create keyed array for quick date access
let length = data.length;
let d, dateParts, date, key;
for(var i=0;i<length;i++) {
d = data[i];
// replace date with actual date object & store in datamap
dateParts = d.Date.split('T')[0].split('-');
date = new Date(dateParts[0], dateParts[1]-1, dateParts[2], 0, 0, 0)
key = date.getFullYear() + "-" + padZero(date.getMonth() + 1) + "-" + padZero(date.getDate());
d.Date = date;
datamap[key] = d;
}

// make sure we have values for each date
let currentDate = startDate;
while(currentDate < endDate) {
key = currentDate.getFullYear() + "-" + padZero(currentDate.getMonth() + 1) + "-" + padZero(currentDate.getDate());
data = datamap[key] ? datamap[key] : {
"Pageviews": 0,
"Visitors": 0,
"Date": new Date(currentDate),
};

newData.push(data);
currentDate.setDate(currentDate.getDate() + 1);
}

return newData;
}



class Chart extends Component {
constructor(props) {
super(props)

this.state = {
loading: false,
data: [],
diffInDays: 1,
hoursPerTick: 24,
}
}

componentWillReceiveProps(newProps, newState) {
if(!this.paramsChanged(this.props, newProps)) {
return;
}

let daysDiff = Math.round((newProps.before-newProps.after)/24/60/60);
let stepHours = daysDiff > 1 ? 24 : 1;
this.setState({
diffInDays: daysDiff,
hoursPerTick: stepHours,
})

this.fetchData(newProps)
}

paramsChanged(o, n) {
return o.siteId != n.siteId || o.before != n.before || o.after != n.after;
}

@bind
prepareData(data) {
let startDate = new Date(this.props.after * 1000);
let endDate = new Date(this.props.before * 1000);
let newData = [];

// instantiate JS Date objects
data = data.map((d) => {
d.Date = new Date(d.Date)
return d
})

// make sure we have values for each date (so 0 value for gaps)
let currentDate = startDate, nextDate, tick, offset = 0;
while(currentDate < endDate) {
tick = {
"Pageviews": 0,
"Visitors": 0,
"Date": new Date(currentDate),
};

nextDate = new Date(currentDate)
nextDate.setHours(nextDate.getHours() + this.state.hoursPerTick);

// grab data that falls between currentDate & nextDate
for(let i=data.length-offset-1; i>=0; i--) {
if( data[i].Date > nextDate) {
break;
}

// increment offset so subsequent dates can skip first X items in array
offset += 1;

// continue to next item in array if we're still below our target date
if( data[i].Date < currentDate) {
continue;
}

// add to tick data
tick.Pageviews += data[i].Pageviews;
tick.Visitors += data[i].Visitors;
}

newData.push(tick);
currentDate = nextDate;
}

return newData;
}



@bind
prepareChart() {
@@ -130,19 +142,28 @@ class Chart extends Component {

this.x = d3.scaleBand().range([0, this.innerWidth]).padding(0.1)
this.y = d3.scaleLinear().range([this.innerHeight, 0])
this.ctx.call(tip)

// tooltip
this.tip = d3.tip().attr('class', 'd3-tip').html((d) => {
let title = this.state.diffInDays <= 1 ? d.Date.toLocaleString() : d.Date.toLocaleDateString();
return (`<div class="tip-heading">${title}</div>
<div class="tip-content">
<div class="tip-pageviews">
<div class="tip-number">${d.Pageviews}</div>
<div class="tip-metric">Pageviews</div>
</div>
<div class="tip-visitors">
<div class="tip-number">${d.Visitors}</div>
<div class="tip-metric">Visitors</div>
</div>
</div>`)});
this.ctx.call(this.tip)
}

@bind
redrawChart() {
let data = this.state.data;

// hide chart & bail if we're trying to show less than 1 day worth of data
this.base.parentNode.style.display = data.length <= 1 ? 'none' : '';
if(data.length <= 1) {
return;
}

if( ! this.ctx ) {
this.prepareChart()
}
@@ -156,8 +177,8 @@ class Chart extends Component {
let yAxis = d3.axisLeft().scale(y).ticks(3).tickSize(-innerWidth)
let xAxis = d3.axisBottom().scale(x).tickFormat(timeFormatPicker(data.length))

// hide all "day" ticks if we're watching more than 100 days of data
if(data.length > 100) {
// hide all "day" ticks if we're watching more than 31 days of data
if(data.length > 31) {
xAxis.tickValues(data.filter(d => d.Date.getDate() === 1).map(d => d.Date))
}

@@ -214,7 +235,7 @@ class Chart extends Component {
.attr('y', (d) => y(d.Visitors))

// add event listeners for tooltips
days.on('mouseover', tip.show).on('mouseout', tip.hide)
days.on('mouseover', this.tip.show).on('mouseout', this.tip.hide)
}

@bind
@@ -228,7 +249,7 @@ class Chart extends Component {
return;
}

let chartData = prepareData(props.after, props.before, d);
let chartData = this.prepareData(d);
this.setState({
loading: false,
data: chartData,
@@ -10,16 +10,6 @@ const padZero = function(n){return n<10? '0'+n:''+n;}

function getNow() {
let now = new Date()
let tzOffset = now.getTimezoneOffset() * 60 * 1000;

// if we're ahead of UTC, stick to UTC's "now"
// this is ugly but a sad necessity for now because we store and aggregate statistics using UTC dates (without time data)
// For those ahead of UTC, "today" will always be empty if they're checking early on in their day
// see https://github.com/usefathom/fathom/issues/134
if (tzOffset < 0) {
now.setTime(now.getTime() + tzOffset )
}

return now
}

@@ -116,8 +106,8 @@ class DatePicker extends Component {

// create unix timestamps from local date objects
let before, after;
before = Math.round((+endDate) / 1000) - endDate.getTimezoneOffset() * 60;
after = Math.round((+startDate) / 1000) - startDate.getTimezoneOffset() * 60;
before = Math.round((+endDate) / 1000);
after = Math.round((+startDate) / 1000);

this.setState({
period: period || '',
@@ -10,7 +10,7 @@ import (
)

func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02T15"))
if stats, ok := r.Sites[cacheKey]; ok {
return stats, nil

@@ -35,7 +35,7 @@ func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*mod
}

func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
if stats, ok := r.Pages[cacheKey]; ok {
return stats, nil
}
@@ -71,7 +71,7 @@ func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostn
}

func (agg *Aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
if stats, ok := r.Referrers[cacheKey]; ok {
return stats, nil
}
@@ -23,10 +23,6 @@ func (api *API) Routes() *mux.Router {

r.Handle("/api/sites/{id:[0-9]+}/stats/site", api.Authorize(HandlerFunc(api.GetSiteStatsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/groupby/day", api.Authorize(HandlerFunc(api.GetSiteStatsPerDayHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/pageviews", api.Authorize(HandlerFunc(api.GetSiteStatsPageviewsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/visitors", api.Authorize(HandlerFunc(api.GetSiteStatsVisitorsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/duration", api.Authorize(HandlerFunc(api.GetSiteStatsDurationHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/bounces", api.Authorize(HandlerFunc(api.GetSiteStatsBouncesHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/realtime", api.Authorize(HandlerFunc(api.GetSiteStatsRealtimeHandler))).Methods(http.MethodGet)

r.Handle("/api/sites/{id:[0-9]+}/stats/pages", api.Authorize(HandlerFunc(api.GetPageStatsHandler))).Methods(http.MethodGet)
@@ -14,46 +14,6 @@ func (api *API) GetSiteStatsHandler(w http.ResponseWriter, r *http.Request) erro
return respond(w, http.StatusOK, envelope{Data: result})
}

// URL: /api/stats/site/pageviews
func (api *API) GetSiteStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetTotalSiteViews(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}

// URL: /api/stats/site/visitors
func (api *API) GetSiteStatsVisitorsHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetTotalSiteVisitors(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}

// URL: /api/stats/site/duration
func (api *API) GetSiteStatsDurationHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetAverageSiteDuration(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}

// URL: /api/stats/site/bounces
func (api *API) GetSiteStatsBouncesHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetAverageSiteBounceRate(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}

// URL: /api/stats/site/realtime
func (api *API) GetSiteStatsRealtimeHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
@@ -30,11 +30,6 @@ type Datastore interface {
GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
SaveSiteStats(*models.SiteStats) error
GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
GetTotalSiteSessions(int64, time.Time, time.Time) (int64, error)
GetAverageSiteDuration(int64, time.Time, time.Time) (float64, error)
GetAverageSiteBounceRate(int64, time.Time, time.Time) (float64, error)
GetRealtimeVisitorCount(int64) (int64, error)

// pageviews
@@ -55,14 +55,13 @@ func (c *Config) DSN() string {
mc.DBName = c.Name
mc.Params = map[string]string{
"parseTime": "true",
"loc": "Local",
}
if c.SSLMode != "" {
mc.Params["tls"] = c.SSLMode
}
dsn = mc.FormatDSN()
case SQLITE:
dsn = c.Name + "?_loc=auto&_busy_timeout=5000"
dsn = c.Name + "?_busy_timeout=5000"
}

return dsn
Oops, something went wrong.

0 comments on commit 2ca1e0f

Please sign in to comment.
You can’t perform that action at this time.