diff --git a/cron/browser_stat.go b/cron/browser_stat.go index f9794bd1a..b5b997bc2 100644 --- a/cron/browser_stat.go +++ b/cron/browser_stat.go @@ -6,7 +6,6 @@ package cron import ( "context" - "database/sql" "strings" "github.com/mssola/user_agent" @@ -27,10 +26,11 @@ func updateBrowserStats(ctx context.Context, hits []goatcounter.Hit) error { return zdb.TX(ctx, func(ctx context.Context, tx zdb.DB) error { // Group by day + browser. type gt struct { - count int - day string - browser string - version string + count int + countUnique int + day string + browser string + version string } grouped := map[string]gt{} for _, h := range hits { @@ -47,21 +47,24 @@ func updateBrowserStats(ctx context.Context, hits []goatcounter.Hit) error { v.browser = browser v.version = version var err error - v.count, err = existingBrowserStats(ctx, tx, h.Site, day, v.browser, v.version) + v.count, v.countUnique, err = existingBrowserStats(ctx, tx, h.Site, day, v.browser, v.version) if err != nil { return err } } v.count += 1 + if h.StartedSession { + v.countUnique += 1 + } grouped[k] = v } siteID := goatcounter.MustGetSite(ctx).ID ins := bulk.NewInsert(ctx, tx, - "browser_stats", []string{"site", "day", "browser", "version", "count"}) + "browser_stats", []string{"site", "day", "browser", "version", "count", "count_unique"}) for _, v := range grouped { - ins.Values(siteID, v.day, v.browser, v.version, v.count) + ins.Values(siteID, v.day, v.browser, v.version, v.count, v.countUnique) } return ins.Finish() }) @@ -70,26 +73,26 @@ func updateBrowserStats(ctx context.Context, hits []goatcounter.Hit) error { func existingBrowserStats( txctx context.Context, tx zdb.DB, siteID int64, day, browser, version string, -) (int, error) { +) (int, int, error) { - var c int - err := tx.GetContext(txctx, &c, - `select count from browser_stats where site=$1 and day=$2 and browser=$3 and version=$4`, + var c []struct { + Count int `db:"count"` + CountUnique int `db:"count_unique"` + } + err := tx.SelectContext(txctx, &c, + `select count, count_unique from browser_stats where site=$1 and day=$2 and browser=$3 and version=$4 limit 1`, siteID, day, browser, version) - if err != nil && err != sql.ErrNoRows { - return 0, errors.Wrap(err, "existing") + if err != nil { + return 0, 0, errors.Wrap(err, "select") } - - if err != sql.ErrNoRows { - _, err = tx.ExecContext(txctx, - `delete from browser_stats where site=$1 and day=$2 and browser=$3 and version=$4`, - siteID, day, browser, version) - if err != nil { - return 0, errors.Wrap(err, "delete") - } + if len(c) == 0 { + return 0, 0, nil } - return c, nil + _, err = tx.ExecContext(txctx, + `delete from browser_stats where site=$1 and day=$2 and browser=$3 and version=$4`, + siteID, day, browser, version) + return c[0].Count, c[0].CountUnique, errors.Wrap(err, "delete") } func getBrowser(uaHeader string) (string, string) { diff --git a/cron/cron.go b/cron/cron.go index 0dcb47cbb..816581729 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "zgo.at/goatcounter" "zgo.at/goatcounter/acme" + "zgo.at/goatcounter/cfg" "zgo.at/utils/syncutil" "zgo.at/zdb" "zgo.at/zhttp/ctxkey" @@ -31,6 +32,7 @@ var tasks = []task{ {DataRetention, 1 * time.Hour}, {renewACME, 2 * time.Hour}, {vacuumDeleted, 12 * time.Hour}, + {clearSessions, 1 * time.Minute}, } var ( @@ -281,3 +283,14 @@ func vacuumDeleted(ctx context.Context) error { } return nil } + +func clearSessions(ctx context.Context) error { + var query string + if cfg.PgSQL { + query = `update sessions set hash=null where last_seen > now() + interval '1 hour'` + } else { + query = `update sessions set hash=null where last_seen > datetime(datetime(), '+1 hours')` + } + _, err := zdb.MustGet(ctx).ExecContext(ctx, query) + return err +} diff --git a/cron/cron_test.go b/cron/cron_test.go index 2102733ba..13bc6b52b 100644 --- a/cron/cron_test.go +++ b/cron/cron_test.go @@ -32,10 +32,10 @@ func TestDataRetention(t *testing.T) { past := now.Add(-40 * 24 * time.Hour) gctest.StoreHits(ctx, t, []goatcounter.Hit{ - {Site: site.ID, CreatedAt: now, Path: "/a"}, - {Site: site.ID, CreatedAt: now, Path: "/a"}, - {Site: site.ID, CreatedAt: past, Path: "/a"}, - {Site: site.ID, CreatedAt: past, Path: "/a"}, + {Session: 1, Site: site.ID, CreatedAt: now, Path: "/a"}, + {Session: 1, Site: site.ID, CreatedAt: now, Path: "/a"}, + {Session: 1, Site: site.ID, CreatedAt: past, Path: "/a"}, + {Session: 1, Site: site.ID, CreatedAt: past, Path: "/a"}, }...) err = DataRetention(ctx) diff --git a/cron/hit_stat.go b/cron/hit_stat.go index efc6a0082..2a407caab 100644 --- a/cron/hit_stat.go +++ b/cron/hit_stat.go @@ -6,7 +6,6 @@ package cron import ( "context" - "database/sql" "strconv" "github.com/pkg/errors" @@ -29,10 +28,11 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error { return zdb.TX(ctx, func(ctx context.Context, tx zdb.DB) error { // Group by day + path. type gt struct { - count []int - day string - path string - title string + count []int + countUnique []int + day string + path string + title string } grouped := map[string]gt{} for _, h := range hits { @@ -47,7 +47,7 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error { v.day = day v.path = h.Path var err error - v.count, v.title, err = existingHitStats(ctx, tx, h.Site, day, v.path) + v.count, v.countUnique, v.title, err = existingHitStats(ctx, tx, h.Site, day, v.path) if err != nil { return err } @@ -57,16 +57,20 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error { v.title = h.Title } - h, _ := strconv.ParseInt(h.CreatedAt.Format("15"), 10, 8) - v.count[h] += 1 + hour, _ := strconv.ParseInt(h.CreatedAt.Format("15"), 10, 8) + v.count[hour] += 1 + if h.StartedSession { + v.count[hour] += 1 + } grouped[k] = v } siteID := goatcounter.MustGetSite(ctx).ID ins := bulk.NewInsert(ctx, tx, - "hit_stats", []string{"site", "day", "path", "title", "stats"}) + "hit_stats", []string{"site", "day", "path", "title", "stats", "stats_unique"}) for _, v := range grouped { - ins.Values(siteID, v.day, v.path, v.title, jsonutil.MustMarshal(v.count)) + ins.Values(siteID, v.day, v.path, v.title, jsonutil.MustMarshal(v.count), + jsonutil.MustMarshal(v.countUnique)) } return ins.Finish() }) @@ -75,37 +79,35 @@ func updateHitStats(ctx context.Context, hits []goatcounter.Hit) error { func existingHitStats( txctx context.Context, tx zdb.DB, siteID int64, day, path string, -) ([]int, string, error) { +) ([]int, []int, string, error) { var ex []struct { - Stats []byte `db:"stats"` - Title string `db:"title"` + Stats []byte `db:"stats"` + StatsUnique []byte `db:"stats_unique"` + Title string `db:"title"` } err := tx.SelectContext(txctx, &ex, - `select stats, title from hit_stats where site=$1 and day=$2 and path=$3`, + `select stats, stats_unique, title from hit_stats where site=$1 and day=$2 and path=$3 limit 1`, siteID, day, path) - if err != nil && err != sql.ErrNoRows { - return nil, "", errors.Wrap(err, "existingHitStats") + if err != nil { + return nil, nil, "", errors.Wrap(err, "existingHitStats") } - if len(ex) == 0 { - return make([]int, 24), "", nil - } - - if len(ex) > 1 { - return nil, "", errors.Errorf("existingHitStats: %d rows: %#v", len(ex), ex) + return make([]int, 24), make([]int, 24), "", nil } _, err = tx.ExecContext(txctx, `delete from hit_stats where site=$1 and day=$2 and path=$3`, siteID, day, path) if err != nil { - return nil, "", errors.Wrap(err, "delete") + return nil, nil, "", errors.Wrap(err, "delete") } - var r []int + var r, ru []int if ex[0].Stats != nil { jsonutil.MustUnmarshal(ex[0].Stats, &r) + jsonutil.MustUnmarshal(ex[0].StatsUnique, &ru) } - return r, ex[0].Title, nil + + return r, ru, ex[0].Title, nil } diff --git a/cron/hit_stat_test.go b/cron/hit_stat_test.go index 3af580099..14fb898d6 100644 --- a/cron/hit_stat_test.go +++ b/cron/hit_stat_test.go @@ -22,9 +22,9 @@ func TestHitStats(t *testing.T) { now := time.Date(2019, 8, 31, 14, 42, 0, 0, time.UTC) goatcounter.Memstore.Append([]goatcounter.Hit{ - {Site: site.ID, CreatedAt: now, Path: "/asd", Title: "aSd"}, - {Site: site.ID, CreatedAt: now, Path: "/asd/"}, // Trailing / should be sanitized and treated identical as /asd - {Site: site.ID, CreatedAt: now, Path: "/zxc"}, + {Session: 1, Site: site.ID, CreatedAt: now, Path: "/asd", Title: "aSd"}, + {Session: 1, Site: site.ID, CreatedAt: now, Path: "/asd/"}, // Trailing / should be sanitized and treated identical as /asd + {Session: 1, Site: site.ID, CreatedAt: now, Path: "/zxc"}, }...) hits, err := goatcounter.Memstore.Persist(ctx) if err != nil { diff --git a/cron/location_stat.go b/cron/location_stat.go index e921f5b6b..e37ee7cd1 100644 --- a/cron/location_stat.go +++ b/cron/location_stat.go @@ -6,7 +6,6 @@ package cron import ( "context" - "database/sql" "github.com/pkg/errors" "zgo.at/goatcounter" @@ -24,9 +23,10 @@ func updateLocationStats(ctx context.Context, hits []goatcounter.Hit) error { return zdb.TX(ctx, func(ctx context.Context, tx zdb.DB) error { // Group by day + location. type gt struct { - count int - day string - location string + count int + countUnique int + day string + location string } grouped := map[string]gt{} for _, h := range hits { @@ -37,21 +37,24 @@ func updateLocationStats(ctx context.Context, hits []goatcounter.Hit) error { v.day = day v.location = h.Location var err error - v.count, err = existingLocationStats(ctx, tx, h.Site, day, v.location) + v.count, v.countUnique, err = existingLocationStats(ctx, tx, h.Site, day, v.location) if err != nil { return err } } v.count += 1 + if h.StartedSession { + v.countUnique += 1 + } grouped[k] = v } siteID := goatcounter.MustGetSite(ctx).ID ins := bulk.NewInsert(ctx, tx, - "location_stats", []string{"site", "day", "location", "count"}) + "location_stats", []string{"site", "day", "location", "count", "count_unique"}) for _, v := range grouped { - ins.Values(siteID, v.day, v.location, v.count) + ins.Values(siteID, v.day, v.location, v.count, v.countUnique) } return ins.Finish() }) @@ -60,24 +63,24 @@ func updateLocationStats(ctx context.Context, hits []goatcounter.Hit) error { func existingLocationStats( txctx context.Context, tx zdb.DB, siteID int64, day, location string, -) (int, error) { +) (int, int, error) { - var c int - err := tx.GetContext(txctx, &c, - `select count from location_stats where site=$1 and day=$2 and location=$3`, + var c []struct { + Count int `db:"count"` + CountUnique int `db:"count_unique"` + } + err := tx.SelectContext(txctx, &c, + `select count, count_unique from location_stats where site=$1 and day=$2 and location=$3`, siteID, day, location) - if err != nil && err != sql.ErrNoRows { - return 0, errors.Wrap(err, "existing") + if err != nil { + return 0, 0, errors.Wrap(err, "select") } - - if err != sql.ErrNoRows { - _, err = tx.ExecContext(txctx, - `delete from location_stats where site=$1 and day=$2 and location=$3`, - siteID, day, location) - if err != nil { - return 0, errors.Wrap(err, "delete") - } + if len(c) == 0 { + return 0, 0, nil } - return c, nil + _, err = tx.ExecContext(txctx, + `delete from location_stats where site=$1 and day=$2 and location=$3`, + siteID, day, location) + return c[0].Count, c[0].CountUnique, errors.Wrap(err, "delete") } diff --git a/cron/ref_stat.go b/cron/ref_stat.go index aa53aab09..2e0d51b3c 100644 --- a/cron/ref_stat.go +++ b/cron/ref_stat.go @@ -6,7 +6,6 @@ package cron import ( "context" - "database/sql" "github.com/pkg/errors" "zgo.at/goatcounter" @@ -24,9 +23,10 @@ func updateRefStats(ctx context.Context, hits []goatcounter.Hit) error { return zdb.TX(ctx, func(ctx context.Context, tx zdb.DB) error { // Group by day + ref. type gt struct { - count int - day string - ref string + count int + countUnique int + day string + ref string } grouped := map[string]gt{} for _, h := range hits { @@ -37,21 +37,24 @@ func updateRefStats(ctx context.Context, hits []goatcounter.Hit) error { v.day = day v.ref = h.Ref var err error - v.count, err = existingRefStats(ctx, tx, h.Site, day, v.ref) + v.count, v.countUnique, err = existingRefStats(ctx, tx, h.Site, day, v.ref) if err != nil { return err } } v.count += 1 + if h.StartedSession { + v.countUnique += 1 + } grouped[k] = v } siteID := goatcounter.MustGetSite(ctx).ID ins := bulk.NewInsert(ctx, tx, - "ref_stats", []string{"site", "day", "ref", "count"}) + "ref_stats", []string{"site", "day", "ref", "count", "count_unique"}) for _, v := range grouped { - ins.Values(siteID, v.day, v.ref, v.count) + ins.Values(siteID, v.day, v.ref, v.count, v.countUnique) } return ins.Finish() }) @@ -60,24 +63,24 @@ func updateRefStats(ctx context.Context, hits []goatcounter.Hit) error { func existingRefStats( txctx context.Context, tx zdb.DB, siteID int64, day, ref string, -) (int, error) { +) (int, int, error) { - var c int - err := tx.GetContext(txctx, &c, - `select count from ref_stats where site=$1 and day=$2 and ref=$3`, + var c []struct { + Count int `db:"count"` + CountUnique int `db:"count_unique"` + } + err := tx.SelectContext(txctx, &c, + `select count, count_unique from ref_stats where site=$1 and day=$2 and ref=$3`, siteID, day, ref) - if err != nil && err != sql.ErrNoRows { - return 0, errors.Wrap(err, "existing") + if err != nil { + return 0, 0, errors.Wrap(err, "select") } - - if err != sql.ErrNoRows { - _, err = tx.ExecContext(txctx, - `delete from ref_stats where site=$1 and day=$2 and ref=$3`, - siteID, day, ref) - if err != nil { - return 0, errors.Wrap(err, "delete") - } + if len(c) == 0 { + return 0, 0, nil } - return c, nil + _, err = tx.ExecContext(txctx, + `delete from ref_stats where site=$1 and day=$2 and ref=$3`, + siteID, day, ref) + return c[0].Count, c[0].CountUnique, errors.Wrap(err, "delete") } diff --git a/cron/size_stat.go b/cron/size_stat.go index ebbeb1c5f..807c5ba86 100644 --- a/cron/size_stat.go +++ b/cron/size_stat.go @@ -6,7 +6,6 @@ package cron import ( "context" - "database/sql" "strconv" "github.com/pkg/errors" @@ -25,9 +24,10 @@ func updateSizeStats(ctx context.Context, hits []goatcounter.Hit) error { return zdb.TX(ctx, func(ctx context.Context, tx zdb.DB) error { // Group by day + width. type gt struct { - count int - day string - width int + count int + countUnique int + day string + width int } grouped := map[string]gt{} for _, h := range hits { @@ -46,21 +46,24 @@ func updateSizeStats(ctx context.Context, hits []goatcounter.Hit) error { v.day = day v.width = width var err error - v.count, err = existingSizeStats(ctx, tx, h.Site, day, v.width) + v.count, v.countUnique, err = existingSizeStats(ctx, tx, h.Site, day, v.width) if err != nil { return err } } v.count += 1 + if h.StartedSession { + v.countUnique += 1 + } grouped[k] = v } siteID := goatcounter.MustGetSite(ctx).ID ins := bulk.NewInsert(ctx, tx, - "size_stats", []string{"site", "day", "width", "count"}) + "size_stats", []string{"site", "day", "width", "count", "count_unique"}) for _, v := range grouped { - ins.Values(siteID, v.day, v.width, v.count) + ins.Values(siteID, v.day, v.width, v.count, v.countUnique) } return ins.Finish() }) @@ -69,24 +72,24 @@ func updateSizeStats(ctx context.Context, hits []goatcounter.Hit) error { func existingSizeStats( txctx context.Context, tx zdb.DB, siteID int64, day string, width int, -) (int, error) { +) (int, int, error) { - var c int - err := tx.GetContext(txctx, &c, - `select count from size_stats where site=$1 and day=$2 and width=$3`, + var c []struct { + Count int `db:"count"` + CountUnique int `db:"count_unique"` + } + err := tx.SelectContext(txctx, &c, + `select count, count_unique from size_stats where site=$1 and day=$2 and width=$3`, siteID, day, width) - if err != nil && err != sql.ErrNoRows { - return 0, errors.Wrap(err, "existing") + if err != nil { + return 0, 0, errors.Wrap(err, "select") } - - if err != sql.ErrNoRows { - _, err = tx.ExecContext(txctx, - `delete from size_stats where site=$1 and day=$2 and width=$3`, - siteID, day, width) - if err != nil { - return 0, errors.Wrap(err, "delete") - } + if len(c) == 0 { + return 0, 0, nil } - return c, nil + _, err = tx.ExecContext(txctx, + `delete from size_stats where site=$1 and day=$2 and width=$3`, + siteID, day, width) + return c[0].Count, c[0].CountUnique, errors.Wrap(err, "delete") } diff --git a/db/migrate/pgsql/2020-03-24-1-sessions.sql b/db/migrate/pgsql/2020-03-24-1-sessions.sql new file mode 100644 index 000000000..27e628cb7 --- /dev/null +++ b/db/migrate/pgsql/2020-03-24-1-sessions.sql @@ -0,0 +1,31 @@ +begin; + create extension pgcrypto; + + create table sessions ( + id serial primary key, + site integer not null check(site > 0), + hash bytea null, + created_at timestamp not null, + last_seen timestamp not null, + + foreign key (site) references sites(id) on delete restrict on update restrict + ); + create unique index "sessions#site#hash" on sessions(site, hash); + + alter table hits add column session int default null; + alter table hits add column started_session int default 0; + + alter table hit_stats add column stats_unique varchar not null default ''; + alter table browser_stats add column count_unique int not null default 0; + alter table location_stats add column count_unique int not null default 0; + alter table ref_stats add column count_unique int not null default 0; + alter table size_stats add column count_unique int not null default 0; + + alter table hit_stats alter column stats_unique drop default; + alter table browser_stats alter column count_unique drop default; + alter table location_stats alter column count_unique drop default; + alter table ref_stats alter column count_unique drop default; + alter table size_stats alter column count_unique drop default; + + insert into version values ('2020-03-24-1-sessions'); +commit; diff --git a/db/migrate/sqlite/2020-03-24-1-sessions.sql b/db/migrate/sqlite/2020-03-24-1-sessions.sql new file mode 100644 index 000000000..1132aee43 --- /dev/null +++ b/db/migrate/sqlite/2020-03-24-1-sessions.sql @@ -0,0 +1,29 @@ +begin; + create table sessions ( + id serial primary key, + site integer not null check(site > 0), + hash bytea null, + created_at timestamp not null, + last_seen timestamp not null, + + foreign key (site) references sites(id) on delete restrict on update restrict + ); + create unique index "sessions#site#hash" on sessions(site, hash); + + alter table hits add column session int default null; + alter table hits add column started_session int default 0; + + alter table hit_stats add column stats_unique varchar not null default ''; + alter table browser_stats add column count_unique int not null default 0; + alter table location_stats add column count_unique int not null default 0; + alter table ref_stats add column count_unique int not null default 0; + alter table size_stats add column count_unique int not null default 0; + + -- alter table hit_stats alter column stats_unique drop default; + -- alter table browser_stats alter column count_unique drop default; + -- alter table location_stats alter column count_unique drop default; + -- alter table ref_stats alter column count_unique drop default; + -- alter table size_stats alter column count_unique drop default; + + insert into version values ('2020-03-24-1-sessions'); +commit; diff --git a/handlers/backend.go b/handlers/backend.go index 3c7e65208..d4cc7785e 100644 --- a/handlers/backend.go +++ b/handlers/backend.go @@ -6,6 +6,7 @@ package handlers import ( "context" + "crypto/sha256" "database/sql" "encoding/csv" "fmt" @@ -244,6 +245,19 @@ func (h backend) count(w http.ResponseWriter, r *http.Request) error { return zhttp.Bytes(w, gif) } + hash := sha256.New() + hash.Write([]byte(fmt.Sprintf("%d%s%s", site.ID, r.UserAgent(), zhttp.RemovePort(r.RemoteAddr)))) + var sess goatcounter.Session + started, err := sess.GetOrCreate(r.Context(), hash.Sum(nil)) + if err != nil { + zlog.Error(err) + //w.Header().Add("X-Goatcounter", fmt.Sprintf("not valid: %s", err)) + w.WriteHeader(500) + return zhttp.Bytes(w, gif) + } + hit.Session = sess.ID + hit.StartedSession = started + err = hit.Validate(r.Context()) if err != nil { w.Header().Add("X-Goatcounter", fmt.Sprintf("not valid: %s", err)) diff --git a/handlers/backend_test.go b/handlers/backend_test.go index a0aa75480..43065e4bc 100644 --- a/handlers/backend_test.go +++ b/handlers/backend_test.go @@ -175,9 +175,9 @@ func TestBackendExport(t *testing.T) { setup: func(ctx context.Context) { now := time.Date(2019, 8, 31, 14, 42, 0, 0, time.UTC) goatcounter.Memstore.Append([]goatcounter.Hit{ - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/zxc", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/zxc", CreatedAt: now}, }...) _, err := goatcounter.Memstore.Persist(ctx) if err != nil { @@ -203,9 +203,9 @@ func TestBackendTpl(t *testing.T) { setup: func(ctx context.Context) { now := time.Date(2019, 8, 31, 14, 42, 0, 0, time.UTC) goatcounter.Memstore.Append([]goatcounter.Hit{ - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/zxc", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/zxc", CreatedAt: now}, }...) _, err := goatcounter.Memstore.Persist(ctx) if err != nil { @@ -252,9 +252,9 @@ func TestBackendPurge(t *testing.T) { setup: func(ctx context.Context) { now := time.Date(2019, 8, 31, 14, 42, 0, 0, time.UTC) goatcounter.Memstore.Append([]goatcounter.Hit{ - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/asd", CreatedAt: now}, - {Site: 1, Path: "/zxc", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/asd", CreatedAt: now}, + {Session: 1, Site: 1, Path: "/zxc", CreatedAt: now}, }...) _, err := goatcounter.Memstore.Persist(ctx) if err != nil { @@ -614,6 +614,7 @@ func TestBackendBarChart(t *testing.T) { }) gctest.StoreHits(ctx, t, goatcounter.Hit{ Site: site.ID, + Session: 1, CreatedAt: tt.hit.UTC(), Path: "/a", }) diff --git a/hit.go b/hit.go index 74fb104de..32a26646b 100644 --- a/hit.go +++ b/hit.go @@ -35,21 +35,23 @@ var ( ) type Hit struct { - ID int64 `db:"id" json:"-"` - Site int64 `db:"site" json:"-"` - - Path string `db:"path" json:"p,omitempty"` - Title string `db:"title" json:"t,omitempty"` - Ref string `db:"ref" json:"r,omitempty"` - Event bool `db:"event" json:"e,omitempty"` - RefParams *string `db:"ref_params" json:"-"` - RefOriginal *string `db:"ref_original" json:"-"` - RefScheme *string `db:"ref_scheme" json:"-"` - Browser string `db:"browser" json:"-"` - Size sqlutil.FloatList `db:"size" json:"s,omitempty"` - Location string `db:"location" json:"-"` - Bot int `db:"bot" json:"-"` - CreatedAt time.Time `db:"created_at" json:"-"` + ID int64 `db:"id" json:"-"` + Site int64 `db:"site" json:"-"` + Session int64 `db:"session" json:"-"` + + Path string `db:"path" json:"p,omitempty"` + Title string `db:"title" json:"t,omitempty"` + Ref string `db:"ref" json:"r,omitempty"` + Event bool `db:"event" json:"e,omitempty"` + Size sqlutil.FloatList `db:"size" json:"s,omitempty"` + + RefParams *string `db:"ref_params" json:"-"` + RefOriginal *string `db:"ref_original" json:"-"` + RefScheme *string `db:"ref_scheme" json:"-"` + Browser string `db:"browser" json:"-"` + Location string `db:"location" json:"-"` + Bot int `db:"bot" json:"-"` + CreatedAt time.Time `db:"created_at" json:"-"` RefURL *url.URL `db:"-" json:"-"` // Parsed Ref UsageDomain string `db:"-" json:"-"` // Track referrer for usage. @@ -298,6 +300,7 @@ func (h *Hit) Validate(ctx context.Context) error { v := zvalidate.New() v.Required("site", h.Site) + v.Required("session", h.Session) v.Required("path", h.Path) v.UTF8("browser", h.Browser) v.UTF8("ref", h.Ref) diff --git a/hit_test.go b/hit_test.go index c1a6a0de3..25397bec0 100644 --- a/hit_test.go +++ b/hit_test.go @@ -42,9 +42,9 @@ func TestHitStatsList(t *testing.T) { }{ { in: []goatcounter.Hit{ - {CreatedAt: hit, Path: "/asd"}, - {CreatedAt: hit.Add(40 * time.Hour), Path: "/asd/"}, - {CreatedAt: hit.Add(100 * time.Hour), Path: "/zxc"}, + {Session: 1, CreatedAt: hit, Path: "/asd"}, + {Session: 1, CreatedAt: hit.Add(40 * time.Hour), Path: "/asd/"}, + {Session: 1, CreatedAt: hit.Add(100 * time.Hour), Path: "/zxc"}, }, wantReturn: "3 3 false ", wantStats: goatcounter.HitStats{ @@ -72,8 +72,8 @@ func TestHitStatsList(t *testing.T) { }, { in: []goatcounter.Hit{ - {CreatedAt: hit, Path: "/asd"}, - {CreatedAt: hit, Path: "/zxc"}, + {Session: 1, CreatedAt: hit, Path: "/asd"}, + {Session: 1, CreatedAt: hit, Path: "/zxc"}, }, inFilter: "x", wantReturn: "1 1 false ", @@ -92,10 +92,10 @@ func TestHitStatsList(t *testing.T) { }, { in: []goatcounter.Hit{ - {CreatedAt: hit, Path: "/a"}, - {CreatedAt: hit, Path: "/aa"}, - {CreatedAt: hit, Path: "/aaa"}, - {CreatedAt: hit, Path: "/aaaa"}, + {Session: 1, CreatedAt: hit, Path: "/a"}, + {Session: 1, CreatedAt: hit, Path: "/aa"}, + {Session: 1, CreatedAt: hit, Path: "/aaa"}, + {Session: 1, CreatedAt: hit, Path: "/aaaa"}, }, inFilter: "a", wantReturn: "4 2 true ", @@ -124,10 +124,10 @@ func TestHitStatsList(t *testing.T) { }, { in: []goatcounter.Hit{ - {CreatedAt: hit, Path: "/a"}, - {CreatedAt: hit, Path: "/aa"}, - {CreatedAt: hit, Path: "/aaa"}, - {CreatedAt: hit, Path: "/aaaa"}, + {Session: 1, CreatedAt: hit, Path: "/a"}, + {Session: 1, CreatedAt: hit, Path: "/aa"}, + {Session: 1, CreatedAt: hit, Path: "/aaa"}, + {Session: 1, CreatedAt: hit, Path: "/aaaa"}, }, inFilter: "a", inExclude: []string{"/aaaa", "/aaa"}, diff --git a/memstore.go b/memstore.go index 979ac1748..b3a022c89 100644 --- a/memstore.go +++ b/memstore.go @@ -49,7 +49,7 @@ func (m *ms) Persist(ctx context.Context) ([]Hit, error) { ins := bulk.NewInsert(ctx, zdb.MustGet(ctx), "hits", []string{"site", "path", "ref", "ref_params", "ref_original", "ref_scheme", "browser", "size", "location", "created_at", "bot", - "title", "event"}) + "title", "event", "session", "started_session"}) usage := bulk.NewInsert(ctx, zdb.MustGet(ctx), "usage", []string{"site", "domain", "count"}) for i, h := range hits { @@ -57,6 +57,7 @@ func (m *ms) Persist(ctx context.Context) ([]Hit, error) { h.RefURL, _ = url.Parse(h.Ref) if h.RefURL != nil { if _, ok := blacklist[h.RefURL.Host]; ok { + zlog.Module("blacklist").Debugf("blacklisted: %q", h.RefURL.Host) continue } } @@ -76,9 +77,13 @@ func (m *ms) Persist(ctx context.Context) ([]Hit, error) { if h.Event { e = 1 } + ss := 0 + if h.StartedSession { + ss = 1 + } ins.Values(h.Site, h.Path, h.Ref, h.RefParams, h.RefOriginal, h.RefScheme, h.Browser, h.Size, h.Location, - h.CreatedAt.Format(zdb.Date), h.Bot, h.Title, e) + h.CreatedAt.Format(zdb.Date), h.Bot, h.Title, e, h.Session, ss) if strings.HasPrefix(h.UsageDomain, "http") { d, err := url.Parse(h.UsageDomain) diff --git a/memstore_test.go b/memstore_test.go index 3e0fea6ff..ab456e152 100644 --- a/memstore_test.go +++ b/memstore_test.go @@ -40,6 +40,7 @@ func gen(ctx context.Context) Hit { s := MustGetSite(ctx) return Hit{ Site: s.ID, + Session: 1, Path: "/test", Ref: "https://example.com/test", Browser: "test", diff --git a/session.go b/session.go new file mode 100644 index 000000000..a5d642965 --- /dev/null +++ b/session.go @@ -0,0 +1,65 @@ +// Copyright © 2019 Martin Tournoij +// This file is part of GoatCounter and published under the terms of the EUPL +// v1.2, which can be found in the LICENSE file or at http://eupl12.zgo.at + +package goatcounter + +import ( + "context" + "database/sql" + "time" + + "github.com/pkg/errors" + "zgo.at/goatcounter/cfg" + "zgo.at/zdb" + "zgo.at/zlog" +) + +type Session struct { + ID int64 `db:"id"` + Site int64 `db:"site"` + + Hash []byte `db:"hash"` + CreatedAt time.Time `db:"created_at"` + LastSeen time.Time `db:"last_seen"` +} + +// GetOrCreate gets the session by hash, creating a new one if it doesn't exist +// yet. +func (s *Session) GetOrCreate(ctx context.Context, hash []byte) (bool, error) { + db := zdb.MustGet(ctx) + site := MustGetSite(ctx) + + err := db.GetContext(ctx, s, `select * from sessions where site=$1 and hash=$2`, site.ID, hash) + switch err { + default: + return false, errors.Wrap(err, "Session.GetOrCreate") + + case nil: + _, err := db.ExecContext(ctx, `update sessions set last_seen=$1 where site=$2 and hash=$3`, + Now().Format(zdb.Date), site.ID, hash) + if err != nil { + zlog.Error(err) + } + return false, nil + + case sql.ErrNoRows: + s.Site = site.ID + s.Hash = hash + s.CreatedAt = Now() + s.LastSeen = Now() + query := `insert into sessions (site, hash, created_at, last_seen) values ($1, $2, $3, $4)` + args := []interface{}{s.Site, s.Hash, s.CreatedAt.Format(zdb.Date), s.LastSeen.Format(zdb.Date)} + if cfg.PgSQL { + err := zdb.MustGet(ctx).GetContext(ctx, &s.ID, query+" returning id", args...) + return true, errors.Wrap(err, "Session.GetOrCreate") + } + + res, err := zdb.MustGet(ctx).ExecContext(ctx, query, args...) + if err != nil { + return true, errors.Wrap(err, "Session.GetOrCreate") + } + s.ID, err = res.LastInsertId() + return true, errors.Wrap(err, "Session.GetOrCreate") + } +}