Show top referrals in totals, rather than a new chart

go.sum

@@ -31,7 +31,6 @@ github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw=
 github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.13.0 h1:LnJI81JidiW9r7pS/hXe6cFeO5EXNq7KbfvoJLRI69c=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/monoculum/formam v0.0.0-20200316225015-49f0baed3a1b h1:BvSZE/bUSz180cQzAEDVOh7seh57UNBlcGAte0CQ8l4=
@@ -102,10 +101,8 @@ zgo.at/utils v0.0.0-20200514044306-bf7c1ff8aa0c h1:GWACHp07xLc1JwEISDzITuw1U4x6+
 zgo.at/utils v0.0.0-20200514044306-bf7c1ff8aa0c/go.mod h1:+PbZTy2300b/A3oAh7/vJY58X5okaK67qKnAzNKJt3k=
 zgo.at/utils v0.0.0-20200524171306-5c9891cdb07f h1:qM6XwtmBUPr1QDR/2L7XqYNnhaOYKXHdG5IvWuGYK2o=
 zgo.at/utils v0.0.0-20200524171306-5c9891cdb07f/go.mod h1:+PbZTy2300b/A3oAh7/vJY58X5okaK67qKnAzNKJt3k=
-zgo.at/zdb v0.0.0-20200518092829-2353fffa61c0 h1:yR9QnKvTQJAkoip/d2YI5l0TX7HG/SvjYsMBVQrYEpw=
-zgo.at/zdb v0.0.0-20200518092829-2353fffa61c0/go.mod h1:rHd8UyafKA8OQCesbWO909CyI4iussi0DRxrcjH5j54=
-zgo.at/zdb v0.0.0-20200527182018-117485d9e261 h1:YlagrnChwFvGsCVkPlufyi6+ZnhjqAof/tVY9931y70=
-zgo.at/zdb v0.0.0-20200527182018-117485d9e261/go.mod h1:rHd8UyafKA8OQCesbWO909CyI4iussi0DRxrcjH5j54=
+zgo.at/zdb v0.0.0-20200527213700-cb59ca25832f h1:TVvrmb7BTBtIBSI3VNltIuplXgSbgDyheB0lJYpdAjg=
+zgo.at/zdb v0.0.0-20200527213700-cb59ca25832f/go.mod h1:rHd8UyafKA8OQCesbWO909CyI4iussi0DRxrcjH5j54=
 zgo.at/zhttp v0.0.0-20200525055724-f2c83b7d82c1 h1:N2Bi3r6RB3THPU2wVajIUi3/AvJ0+5joRomVjByUz+c=
 zgo.at/zhttp v0.0.0-20200525055724-f2c83b7d82c1/go.mod h1:lnKdr+Z/u747AVwOkBKweLbFFe0yyvV2JY8NONI4XpQ=
 zgo.at/zlog v0.0.0-20200404052423-adffcc8acd57 h1:sIuh9IpccNlzMsmAvF7EPT87/Ab2XdUJHizsMxbQgM4=

handlers/backend.go

@@ -145,7 +145,6 @@ func (h backend) Mount(r chi.Router, db zdb.DB) {
 			ap.Get("/systems", zhttp.Wrap(h.systems))
 			ap.Get("/sizes", zhttp.Wrap(h.sizes))
 			ap.Get("/locations", zhttp.Wrap(h.locations))
-			ap.Get("/toprefs", zhttp.Wrap(h.topRefs))
 			ap.Get("/pages-by-ref", zhttp.Wrap(h.pagesByRef))
 		}
 		{
@@ -375,18 +374,18 @@ func (h backend) index(w http.ResponseWriter, r *http.Request) error {
 	}
 	showMoreLoc := len(locStat) > 0 && float32(locStat[len(locStat)-1].Count)/float32(totalLoc)*100 < 3.0
 
-	var topRefs goatcounter.Stats
-	totalTopRefs, showMoreRefs, err := topRefs.ListRefs(r.Context(), start, end, 10, 0)
-	if err != nil {
-		return err
-	}
-
 	// Add refers.
 	sr := r.URL.Query().Get("showrefs")
-	var refs goatcounter.HitStats
-	var moreRefs bool
+	var (
+		refs     goatcounter.HitStats
+		moreRefs bool
+	)
 	if sr != "" {
-		moreRefs, err = refs.ListRefs(r.Context(), sr, start, end, 0)
+		if sr == goatcounter.PathTotals {
+			moreRefs, err = refs.ListAllRefs(r.Context(), start, end, 0)
+		} else {
+			moreRefs, err = refs.ListRefs(r.Context(), sr, start, end, 0)
+		}
 		if err != nil {
 			return err
 		}
@@ -444,44 +443,18 @@ func (h backend) index(w http.ResponseWriter, r *http.Request) error {
 		LocationStat       goatcounter.Stats
 		TotalLocation      int
 		ShowMoreLocations  bool
-		TopRefs            goatcounter.Stats
-		TotalTopRefs       int
-		ShowMoreRefs       bool
 		Daily              bool
 		ForcedDaily        bool
 		Max                int
 	}{newGlobals(w, r), cd, sr, r.URL.Query().Get("hl-period"), start, end,
 		filter, pages, totalPages, morePages, refs, moreRefs, total,
 		totalUnique, totalDisplay, totalUniqueDisplay, browsers, totalBrowsers,
 		systems, totalSystems, subs, sizeStat, totalSize, locStat, totalLoc,
-		showMoreLoc, topRefs, totalTopRefs, showMoreRefs, daily, forcedDaily,
-		max})
+		showMoreLoc, daily, forcedDaily, max})
 	l.Since("zhttp.Template")
 	return x
 }
 
-func (h backend) topRefs(w http.ResponseWriter, r *http.Request) error {
-	start, end, err := getPeriod(w, r, goatcounter.MustGetSite(r.Context()))
-	if err != nil {
-		return err
-	}
-
-	var refs goatcounter.Stats
-	o, _ := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
-	total, hasMore, err := refs.ListRefs(r.Context(), start, end, 10, int(o))
-	if err != nil {
-		return err
-	}
-
-	t, _ := strconv.ParseInt(r.URL.Query().Get("total"), 10, 64)
-	tpl := goatcounter.HorizontalChart(r.Context(), refs, total, int(t), 0, true, false)
-
-	return zhttp.JSON(w, map[string]interface{}{
-		"html":     string(tpl),
-		"has_more": hasMore,
-	})
-}
-
 func (h backend) pagesByRef(w http.ResponseWriter, r *http.Request) error {
 	start, end, err := getPeriod(w, r, goatcounter.MustGetSite(r.Context()))
 	if err != nil {
@@ -516,10 +489,22 @@ func (h backend) refs(w http.ResponseWriter, r *http.Request) error {
 		offset = int(o2)
 	}
 
-	var refs goatcounter.HitStats
-	more, err := refs.ListRefs(r.Context(), r.URL.Query().Get("showrefs"), start, end, offset)
-	if err != nil {
-		return err
+	showRefs := r.URL.Query().Get("showrefs")
+
+	var (
+		refs goatcounter.HitStats
+		more bool
+	)
+	if showRefs == goatcounter.PathTotals {
+		more, err = refs.ListAllRefs(r.Context(), start, end, offset)
+		if err != nil {
+			return err
+		}
+	} else {
+		more, err = refs.ListRefs(r.Context(), showRefs, start, end, offset)
+		if err != nil {
+			return err
+		}
 	}
 
 	tpl, err := zhttp.ExecuteTpl("_backend_refs.gohtml", map[string]interface{}{

hit.go

@@ -481,6 +481,9 @@ func (h *HitStats) ListRefs(ctx context.Context, path string, start, end time.Ti
 		order by count desc, path desc
 		limit $5 offset $6`,
 		site.ID, path, start.Format(zdb.Date), end.Format(zdb.Date), limit+1, offset)
+	if err != nil {
+		errors.Wrap(err, "RefStats.ListRefs")
+	}
 
 	var more bool
 	if len(*h) > limit {
@@ -490,7 +493,56 @@ func (h *HitStats) ListRefs(ctx context.Context, path string, start, end time.Ti
 		*h = hh
 	}
 
-	return more, errors.Wrap(err, "RefStats.ListRefs")
+	return more, nil
+}
+
+// ListAllRefs lists all ref statistics for the given time period, excluding
+// referrals from the configured LinkDomain.
+//
+// The returned count is the count without LinkDomain, and is different from the
+// total number of hits.
+//
+// TODO: after ref_counts it no longer lists "unknown".
+func (h *HitStats) ListAllRefs(ctx context.Context, start, end time.Time, offset int) (bool, error) {
+	site := MustGetSite(ctx)
+
+	limit := site.Settings.Limits.Ref
+	if limit == 0 {
+		limit = 10
+	}
+
+	where := ` where site=? and hour>=? and hour<=?`
+	args := []interface{}{site.ID, start.Format(zdb.Date), end.Format(zdb.Date)}
+	if site.LinkDomain != "" {
+		where += " and ref not like ? "
+		args = append(args, site.LinkDomain+"%")
+	}
+
+	db := zdb.MustGet(ctx)
+	err := db.SelectContext(ctx, h, db.Rebind(`/* HitStats.ListAllRefs */
+		select
+			coalesce(sum(total), 0) as count,
+			coalesce(sum(total_unique), 0) as count_unique,
+			max(ref_scheme) as ref_scheme,
+			ref as path
+		from ref_counts`+
+		where+`
+		group by ref
+		order by count desc
+		limit ? offset ?`), append(args, limit+1, offset)...)
+	if err != nil {
+		return false, errors.Wrap(err, "HitStats.ListAllRefs")
+	}
+
+	var more bool
+	if len(*h) > limit {
+		more = true
+		hh := *h
+		hh = hh[:len(hh)-1]
+		*h = hh
+	}
+
+	return more, nil
 }
 
 // ListPaths lists all paths we have statistics for.
@@ -542,58 +594,6 @@ func (h *Stats) ByRef(ctx context.Context, start, end time.Time, ref string) (in
 	return total, errors.Wrap(err, "HitStats.ByRef")
 }
 
-// ListRefs lists all ref statistics for the given time period, excluding
-// referrals from the configured LinkDomain.
-//
-// The returned count is the count without LinkDomain, and is different from the
-// total number of hits.
-//
-// TODO: after ref_counts it no longer lists "unknown".
-func (h *Stats) ListRefs(ctx context.Context, start, end time.Time, limit, offset int) (int, bool, error) {
-	site := MustGetSite(ctx)
-
-	where := ` where site=? and hour>=? and hour<=?`
-	args := []interface{}{site.ID, start.Format(zdb.Date), end.Format(zdb.Date)}
-	if site.LinkDomain != "" {
-		where += " and ref not like ? "
-		args = append(args, site.LinkDomain+"%")
-	}
-
-	db := zdb.MustGet(ctx)
-	err := db.SelectContext(ctx, h, db.Rebind(`/* Stats.ListRefs */
-		select
-			ref as name,
-			coalesce(sum(total), 0) as count,
-			coalesce(sum(total_unique), 0) as count_unique
-		from ref_counts`+
-		where+`
-		group by ref
-		order by count desc
-		limit ? offset ?`), append(args, limit+1, offset)...)
-	if err != nil {
-		return 0, false, errors.Wrap(err, "Stats.ListRefs")
-	}
-
-	// TODO: unique totals
-	var total int
-	err = db.GetContext(ctx, &total,
-		db.Rebind(`select coalesce(sum(total), 0) from ref_counts`+where),
-		args...)
-	if err != nil {
-		return 0, false, errors.Wrap(err, "Stats.ListRefs: total")
-	}
-
-	var more bool
-	if len(*h) > limit {
-		more = true
-		hh := *h
-		hh = hh[:len(hh)-1]
-		*h = hh
-	}
-
-	return total, more, nil
-}
-
 // ListBrowsers lists all browser statistics for the given time period.
 func (h *Stats) ListBrowsers(ctx context.Context, start, end time.Time) (int, error) {
 	err := zdb.MustGet(ctx).SelectContext(ctx, h, `/* Stats.ListBrowsers */

hit_list.go

@@ -184,6 +184,11 @@ func (h *HitStats) List(
 	return total, totalUnique, totalDisplay, totalUniqueDisplay, more, nil
 }
 
+// PathTotals is a special path to indicate this is the "total" overview.
+//
+// Trailing whitespace is trimmed on paths, so this should never conflict.
+const PathTotals = "TOTAL "
+
 // Totals gets the totals overview of all pages.
 func (h *HitStat) Totals(ctx context.Context, start, end time.Time, filter string, daily bool) (int, error) {
 	l := zlog.Module("HitStat.Totals")
@@ -211,7 +216,7 @@ func (h *HitStat) Totals(ctx context.Context, start, end time.Time, filter strin
 	l = l.Since("total overview")
 
 	totalst := HitStat{
-		Path:  "",
+		Path:  PathTotals,
 		Title: "",
 	}
 	stats := make(map[string]Stat)