Skip to content
Permalink
Browse files

Persist and run stats at the same time

Not a fix for #39, but does lessen the impact.
  • Loading branch information...
arp242 committed Sep 16, 2019
1 parent 53ee908 commit 1bd92cf658b23cce416bb5c3a36f2347196158d8
Showing with 31 additions and 18 deletions.
  1. +20 −7 cron/cron.go
  2. +1 −1 go.mod
  3. +2 −0 go.sum
  4. +8 −10 memstore.go
@@ -23,8 +23,9 @@ type task struct {
}

var tasks = []task{
{goatcounter.Memstore.Persist, 10 * time.Second},
{updateStats, 60 * time.Second},
//{goatcounter.Memstore.Persist, 10 * time.Second},
//{updateStats, 60 * time.Second},
{persistAndStat, 10 * time.Second},
}

var wg sync.WaitGroup
@@ -68,20 +69,33 @@ func Run(db *sqlx.DB) {
// on shutdown.
func Wait(db *sqlx.DB) {
ctx := context.WithValue(context.Background(), ctxkey.DB, db)
l := zlog.Module("cron")

wg.Wait()

for _, t := range tasks {
err := t.fun(ctx)
if err != nil {
l.Error(err)
zlog.Module("cron").Error(err)
}
}
}
func updateStats(ctx context.Context) error {
l := zlog.SetDebug("stat").Module("stat")

func persistAndStat(ctx context.Context) error {
l := zlog.Module("cron")

hl := goatcounter.Memstore.Len()
err := goatcounter.Memstore.Persist(ctx)
if err != nil {
return err
}
l = l.Since("memstore")

err = updateStats(ctx)
l.Since("stats").FieldsSince().Printf("persisted %d hits", hl)
return err
}

func updateStats(ctx context.Context) error {
var sites goatcounter.Sites
err := sites.List(ctx)
if err != nil {
@@ -109,6 +123,5 @@ func updateStats(ctx context.Context) error {
return errors.Wrapf(err, "update last_stat: site %d", s.ID)
}
}
l.Since("updateAllStats")
return nil
}
2 go.mod
@@ -24,7 +24,7 @@ require (
golang.org/x/sys v0.0.0-20190830080133-08d80c9d36de // indirect
google.golang.org/appengine v1.6.2 // indirect
zgo.at/zhttp v0.0.0-20190915115045-e8088560373f
zgo.at/zlog v1.0.0
zgo.at/zlog v1.0.1
zgo.at/zlog_sentry v1.0.1
)

2 go.sum
@@ -86,5 +86,7 @@ zgo.at/zhttp v0.0.0-20190915115045-e8088560373f h1:7zC/B5ERRABKPTw50ls4MAGRpQVtE
zgo.at/zhttp v0.0.0-20190915115045-e8088560373f/go.mod h1:MqXfkj0gJ3kD3N7Iw20QquA2ShVFluU7BFm5shEnhLM=
zgo.at/zlog v1.0.0 h1:5r5ARbp6hbeMB1AmpppJZIxlRfE0Fp0OzqNlxVFagvQ=
zgo.at/zlog v1.0.0/go.mod h1:/+tHzt+nRogoJkSjzSgmd9OeDFlTbjKnQUuEkald8pw=
zgo.at/zlog v1.0.1 h1:jK4gxr49m5GP8O5ay8QJFZ7qINvVRG0m7eW6z8qg51Q=
zgo.at/zlog v1.0.1/go.mod h1:/+tHzt+nRogoJkSjzSgmd9OeDFlTbjKnQUuEkald8pw=
zgo.at/zlog_sentry v1.0.1 h1:fTEzWrPaZIuu3qB9udlCG5JJjhxcrv/p2DcmGyY0t6o=
zgo.at/zlog_sentry v1.0.1/go.mod h1:uCJZMOY7eFNBmsqKdQz7UbYc6TmnmB7IWmwau86lbr8=
@@ -6,7 +6,6 @@ package goatcounter

import (
"context"
"fmt"
"net/url"
"sync"

@@ -29,13 +28,18 @@ func (m *ms) Append(hit Hit) {
m.Unlock()
}

func (m *ms) Len() int {
m.Lock()
l := len(m.hits)
m.Unlock()
return l
}

func (m *ms) Persist(ctx context.Context) error {
if len(m.hits) == 0 {
return nil
}

l := zlog.SetDebug("memstore").Module("memstore")

m.Lock()
hits := make([]Hit, len(m.hits))
copy(hits, m.hits)
@@ -68,11 +72,5 @@ func (m *ms) Persist(ctx context.Context) error {
ins.Values(h.Site, h.Path, h.Ref, h.RefParams, h.RefOriginal,
h.RefScheme, h.Browser, h.Size, sqlDate(h.CreatedAt))
}
err := ins.Finish()
if err != nil {
zlog.Error(err)
}

l.Since(fmt.Sprintf("persisted %d hits", len(hits)))
return nil
return ins.Finish()
}

0 comments on commit 1bd92cf

Please sign in to comment.
You can’t perform that action at this time.