From 2c063f5905cbb5ad6a977a2d3e6bde82e52d5679 Mon Sep 17 00:00:00 2001
From: Seth Vargo <seth@sethvargo.com>
Date: Wed, 2 Dec 2020 16:36:22 -0500
Subject: [PATCH] Capture and display daily active user stats

---
 cmd/apiserver/main.go                  | 13 +---
 cmd/server/assets/realmadmin/show.html | 99 +++++++++++++++++++-------
 docs/api.md                            |  6 ++
 pkg/controller/middleware/chaff.go     | 77 ++++++++++++++++++++
 pkg/database/migrations.go             | 11 +++
 pkg/database/realm.go                  | 21 +++++-
 pkg/database/realm_stats.go            | 31 ++++----
 7 files changed, 207 insertions(+), 51 deletions(-)
 create mode 100644 pkg/controller/middleware/chaff.go

diff --git a/cmd/apiserver/main.go b/cmd/apiserver/main.go
index 33a152ed3..b1ed8fb21 100644
--- a/cmd/apiserver/main.go
+++ b/cmd/apiserver/main.go
@@ -20,7 +20,6 @@ import (
 	"context"
 	"crypto/sha1"
 	"fmt"
-	"net/http"
 	"os"
 	"strconv"
 
@@ -182,7 +181,7 @@ func realMain(ctx context.Context) error {
 		sub := r.PathPrefix("/api/verify").Subrouter()
 		sub.Use(requireAPIKey)
 		sub.Use(processFirewall)
-		sub.Use(processChaff(verifyChaffTracker))
+		sub.Use(middleware.ProcessChaff(db, verifyChaffTracker))
 		sub.Use(rateLimit)
 
 		// POST /api/verify
@@ -197,7 +196,7 @@ func realMain(ctx context.Context) error {
 		sub := r.PathPrefix("/api/certificate").Subrouter()
 		sub.Use(requireAPIKey)
 		sub.Use(processFirewall)
-		sub.Use(processChaff(certChaffTracker))
+		sub.Use(middleware.ProcessChaff(db, certChaffTracker))
 		sub.Use(rateLimit)
 
 		// POST /api/certificate
@@ -216,14 +215,6 @@ func realMain(ctx context.Context) error {
 	return srv.ServeHTTPHandler(ctx, handlers.CombinedLoggingHandler(os.Stdout, r))
 }
 
-func processChaff(t *chaff.Tracker) mux.MiddlewareFunc {
-	detector := chaff.HeaderDetector("X-Chaff")
-
-	return func(next http.Handler) http.Handler {
-		return t.HandleTrack(detector, next)
-	}
-}
-
 // makePadFromChaff makes a Padding structure from chaff data.
 // Note, the random chaff data will be longer than necessary, so we shorten it.
 func makePadFromChaff(s string) api.Padding {
diff --git a/cmd/server/assets/realmadmin/show.html b/cmd/server/assets/realmadmin/show.html
index 1e0a7080f..4f7829090 100644
--- a/cmd/server/assets/realmadmin/show.html
+++ b/cmd/server/assets/realmadmin/show.html
@@ -36,6 +36,23 @@ <h1>Realm stats</h1>
       </small>
     </div>
 
+    <div class="card mb-3">
+      <div class="card-header">
+        <span class="oi oi-bar-chart mr-2 ml-n1"></span>
+        Daily active users
+        <span class="font-weight-bold float-right" data-toggle="tooltip"
+          title="These are daily active users as reported by the apps. See the verification server API documentation for how to populate these metrics.">?</span>
+      </div>
+      <div id="daily_active_users_chart" class="container d-flex h-100 w-100" style="min-height:400px;">
+        <p class="justify-content-center align-self-center text-center font-italic w-100">Loading chart...</p>
+      </div>
+      <small class="card-footer text-muted text-right">
+        <span class="mr-1">Export as:</span>
+        <a href="/realm/stats.csv" class="mr-1">CSV</a>
+        <a href="/realm/stats.json">JSON</a>
+      </small>
+    </div>
+
     <div class="row">
       <div class="col-lg-6 pr-2">
         <div class="card mb-3">
@@ -84,6 +101,7 @@ <h1>Realm stats</h1>
   <script src="https://www.gstatic.com/charts/loader.js"></script>
   <script>
     let realmChartDiv = document.getElementById('realm_chart');
+    let dailyUsersChartDiv = document.getElementById('daily_active_users_chart');
     let $perUserTable = $('#per_user_table');
     let $perExternalIssuerTable = $('#per_external_issuer_table');
     let dateFormatter;
@@ -100,9 +118,9 @@ <h1>Realm stats</h1>
     });
 
     function drawCharts() {
-      drawRealmChart();
-      drawUsersChart();
-      drawExternalIssuersChart();
+      drawRealmCharts();
+      drawUsersTable();
+      drawExternalIssuersTable();
     }
 
     // utcDate parses the given RFC-3339 date as a javascript date, then
@@ -113,7 +131,7 @@ <h1>Realm stats</h1>
       return new Date(d.getTime() + offset);
     }
 
-    function drawRealmChart() {
+    function drawRealmCharts() {
       $.ajax({
         url: '/realm/stats.json',
         dataType: 'json',
@@ -123,38 +141,67 @@ <h1>Realm stats</h1>
           return;
         }
 
-        var dataTable = new google.visualization.DataTable();
+        // Code stats
+        {
+          var dataTable = new google.visualization.DataTable();
+          dataTable.addColumn('date', 'Date');
+          dataTable.addColumn('number', 'Issued');
+          dataTable.addColumn('number', 'Claimed');
 
-        dataTable.addColumn('date', 'Date');
-        dataTable.addColumn('number', 'Issued');
-        dataTable.addColumn('number', 'Claimed');
+          data.statistics.reverse().forEach(function(row) {
+            dataTable.addRow([utcDate(row.date), row.data.codes_issued, row.data.codes_claimed]);
+          });
 
-        data.statistics.reverse().forEach(function(row) {
-          dataTable.addRow([utcDate(row.date), row.data.codes_issued, row.data.codes_claimed]);
-        });
+          dateFormatter.format(dataTable, 0);
+
+          let options = {
+            colors: ['#007bff', '#ff7b00'],
+            chartArea: {
+              left: 30, // leave room for y-axis labels
+              width: '100%'
+            },
+            hAxis: { format: 'M/d' },
+            legend: { position: 'top' },
+            width: '100%'
+          };
+
+          let chart = new google.visualization.LineChart(realmChartDiv);
+          chart.draw(dataTable, options);
+        }
+
+        // Daily actives
+        {
+          var dataTable = new google.visualization.DataTable();
+          dataTable.addColumn('date', 'Date');
+          dataTable.addColumn('number', 'Users');
+
+          data.statistics.reverse().forEach(function(row) {
+            dataTable.addRow([utcDate(row.date), row.data.daily_active_users]);
+          });
 
-        dateFormatter.format(dataTable, 0);
+          dateFormatter.format(dataTable, 0);
 
-        let options = {
-          colors: ['#007bff', '#ff7b00'],
-          chartArea: {
-            left: 30, // leave room for y-axis labels
+          let options = {
+            colors: ['#007bff', '#ff7b00'],
+            chartArea: {
+              left: 30, // leave room for y-axis labels
+              width: '100%'
+            },
+            hAxis: { format: 'M/d' },
+            legend: 'none',
             width: '100%'
-          },
-          hAxis: { format: 'M/d' },
-          legend: { position: 'top' },
-          width: '100%'
-        };
-
-        let chart = new google.visualization.LineChart(realmChartDiv);
-        chart.draw(dataTable, options);
+          };
+
+          let chart = new google.visualization.LineChart(dailyUsersChartDiv);
+          chart.draw(dataTable, options);
+        }
       })
       .fail(function(xhr, status, err) {
         flash.error('Failed to render realm stats: ' + err);
       });
     }
 
-    function drawUsersChart() {
+    function drawUsersTable() {
       $.ajax({
         url: '/realm/stats.json',
         data: { scope: 'user' },
@@ -233,7 +280,7 @@ <h1>Realm stats</h1>
       });
     }
 
-    function drawExternalIssuersChart() {
+    function drawExternalIssuersTable() {
       $.ajax({
         url: '/realm/stats.json',
         data: { scope: 'external' },
diff --git a/docs/api.md b/docs/api.md
index 04ec58661..cd06bd494 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -338,6 +338,12 @@ curl https://apiserver.example.com/api/verify \
   --data '{"padding":"base64 encoded padding"}'
 ```
 
+If the `X-Chaff` header has a value of "daily" (case-insensitive), the request
+will be counted toward the realm's daily active user count. Since the server
+records data in UTC time, clients should send this value once **per UTC day** if
+they are collecting server-side adoption metrics. The server will still respond
+with fake data that should be ignored per guidance below.
+
 The client should still send a real request with a real request body (the body
 will not be processed). The server will respond with a fake response that your
 client **MUST NOT** process or parse. The response will not be a valid JSON
diff --git a/pkg/controller/middleware/chaff.go b/pkg/controller/middleware/chaff.go
new file mode 100644
index 000000000..8a21041c7
--- /dev/null
+++ b/pkg/controller/middleware/chaff.go
@@ -0,0 +1,77 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middleware
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-verification-server/internal/project"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller"
+	"github.com/google/exposure-notifications-verification-server/pkg/database"
+	"github.com/gorilla/mux"
+	"github.com/mikehelmick/go-chaff"
+)
+
+const (
+	// ChaffHeader is the chaff header key.
+	ChaffHeader = "X-Chaff"
+
+	// ChaffDailyKey is the key to check if the chaff should be counted toward
+	// daily stats.
+	ChaffDailyKey = "daily"
+)
+
+// ProcessChaff injects the chaff processing middleware. If chaff requests send
+// a value of "daily" (case-insensitive), they will be counted toward the
+// realm's total active users and return a chaff response. Any other values will
+// only return a chaff response.
+//
+// This must come after RequireAPIKey.
+func ProcessChaff(db *database.Database, t *chaff.Tracker) mux.MiddlewareFunc {
+	detector := chaff.HeaderDetector(ChaffHeader)
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx := r.Context()
+			now := time.Now().UTC()
+
+			logger := logging.FromContext(ctx).Named("middleware.ProcessChaff")
+
+			chaffValue := project.TrimSpace(strings.ToLower(r.Header.Get(ChaffHeader)))
+			if chaffValue == ChaffDailyKey {
+				currentRealm := controller.RealmFromContext(ctx)
+				if currentRealm == nil {
+					logger.Error("missing current realm in context")
+				} else {
+					// Increment DAU asynchronously and out-of-band for the request. These
+					// statistics are best-effort and we don't want to block or delay
+					// rendering to populate stats.
+					go func() {
+						if err := currentRealm.IncrementDailyActiveUsers(db, now); err != nil {
+							logger.Errorw("failed to increment daily active stats",
+								"realm", currentRealm.ID,
+								"error", err)
+						}
+					}()
+				}
+			}
+
+			t.HandleTrack(detector, next).ServeHTTP(w, r)
+		})
+	}
+}
diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go
index bfd444003..915eeb1f5 100644
--- a/pkg/database/migrations.go
+++ b/pkg/database/migrations.go
@@ -1725,6 +1725,17 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate {
 				return tx.DropTable(&ExternalIssuerStat{}).Error
 			},
 		},
+		{
+			ID: "00071-AddDailyActiveUsersToRealmStats",
+			Migrate: func(tx *gorm.DB) error {
+				sql := `ALTER TABLE realm_stats ADD COLUMN IF NOT EXISTS daily_active_users INTEGER DEFAULT 0`
+				return tx.Exec(sql).Error
+			},
+			Rollback: func(tx *gorm.DB) error {
+				sql := `ALTER TABLE realm_stats DROP COLUMN IF EXISTS daily_active_users`
+				return tx.Exec(sql).Error
+			},
+		},
 	})
 }
 
diff --git a/pkg/database/realm.go b/pkg/database/realm.go
index 5e23edf7d..a2d262664 100644
--- a/pkg/database/realm.go
+++ b/pkg/database/realm.go
@@ -1267,7 +1267,8 @@ func (r *Realm) Stats(db *Database, start, stop time.Time) (RealmStats, error) {
 			d.date AS date,
 			$1 AS realm_id,
 			COALESCE(s.codes_issued, 0) AS codes_issued,
-			COALESCE(s.codes_claimed, 0) AS codes_claimed
+			COALESCE(s.codes_claimed, 0) AS codes_claimed,
+			COALESCE(s.daily_active_users, 0) AS daily_active_users
 		FROM (
 			SELECT date::date FROM generate_series($2, $3, '1 day'::interval) date
 		) d
@@ -1394,6 +1395,24 @@ func (r *Realm) QuotaKey(hmacKey []byte) (string, error) {
 	return fmt.Sprintf("realm:quota:%s", dig), nil
 }
 
+// IncrementDailyActiveUsers increments the daily active users for the realm by
+// the provided amount.
+func (r *Realm) IncrementDailyActiveUsers(db *Database, now time.Time) error {
+	date := timeutils.Midnight(now)
+
+	sql := `
+		INSERT INTO realm_stats (date, realm_id, daily_active_users)
+			VALUES ($1, $2, 1)
+		ON CONFLICT (date, realm_id) DO UPDATE
+			SET daily_active_users = realm_stats.daily_active_users + 1
+	`
+
+	if err := db.db.Exec(sql, date, r.ID).Error; err != nil {
+		return fmt.Errorf("failed to increment daily active users for realm %d: %w", r.ID, err)
+	}
+	return nil
+}
+
 // ToCIDRList converts the newline-separated and/or comma-separated CIDR list
 // into an array of strings.
 func ToCIDRList(s string) ([]string, error) {
diff --git a/pkg/database/realm_stats.go b/pkg/database/realm_stats.go
index 8f5f85eda..33c0d848d 100644
--- a/pkg/database/realm_stats.go
+++ b/pkg/database/realm_stats.go
@@ -33,10 +33,11 @@ type RealmStats []*RealmStat
 
 // RealmStat represents statistics related to a user in the database.
 type RealmStat struct {
-	Date         time.Time `gorm:"date; not null;"`
-	RealmID      uint      `gorm:"realm_id; not null;"`
-	CodesIssued  uint      `gorm:"codes_issued; default:0;"`
-	CodesClaimed uint      `gorm:"codes_claimed; default:0;"`
+	Date             time.Time `gorm:"date; not null;"`
+	RealmID          uint      `gorm:"realm_id; not null;"`
+	CodesIssued      uint      `gorm:"codes_issued; default:0;"`
+	CodesClaimed     uint      `gorm:"codes_claimed; default:0;"`
+	DailyActiveUsers uint      `gorm:"daily_active_users; default:0;"`
 }
 
 // MarshalCSV returns bytes in CSV format.
@@ -49,7 +50,7 @@ func (s RealmStats) MarshalCSV() ([]byte, error) {
 	var b bytes.Buffer
 	w := csv.NewWriter(&b)
 
-	if err := w.Write([]string{"date", "codes_issued", "codes_claimed"}); err != nil {
+	if err := w.Write([]string{"date", "codes_issued", "codes_claimed", "daily_active_users"}); err != nil {
 		return nil, fmt.Errorf("failed to write CSV header: %w", err)
 	}
 
@@ -58,6 +59,7 @@ func (s RealmStats) MarshalCSV() ([]byte, error) {
 			stat.Date.Format("2006-01-02"),
 			strconv.FormatUint(uint64(stat.CodesIssued), 10),
 			strconv.FormatUint(uint64(stat.CodesClaimed), 10),
+			strconv.FormatUint(uint64(stat.DailyActiveUsers), 10),
 		}); err != nil {
 			return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err)
 		}
@@ -82,8 +84,9 @@ type jsonRealmStatStats struct {
 }
 
 type jsonRealmStatStatsData struct {
-	CodesIssued  uint `json:"codes_issued"`
-	CodesClaimed uint `json:"codes_claimed"`
+	CodesIssued      uint `json:"codes_issued"`
+	CodesClaimed     uint `json:"codes_claimed"`
+	DailyActiveUsers uint `json:"daily_active_users"`
 }
 
 // MarshalJSON is a custom JSON marshaller.
@@ -98,8 +101,9 @@ func (s RealmStats) MarshalJSON() ([]byte, error) {
 		stats = append(stats, &jsonRealmStatStats{
 			Date: stat.Date,
 			Data: &jsonRealmStatStatsData{
-				CodesIssued:  stat.CodesIssued,
-				CodesClaimed: stat.CodesClaimed,
+				CodesIssued:      stat.CodesIssued,
+				CodesClaimed:     stat.CodesClaimed,
+				DailyActiveUsers: stat.DailyActiveUsers,
 			},
 		})
 	}
@@ -132,10 +136,11 @@ func (s *RealmStats) UnmarshalJSON(b []byte) error {
 
 	for _, stat := range result.Stats {
 		*s = append(*s, &RealmStat{
-			Date:         stat.Date,
-			RealmID:      result.RealmID,
-			CodesIssued:  stat.Data.CodesIssued,
-			CodesClaimed: stat.Data.CodesClaimed,
+			Date:             stat.Date,
+			RealmID:          result.RealmID,
+			CodesIssued:      stat.Data.CodesIssued,
+			CodesClaimed:     stat.Data.CodesClaimed,
+			DailyActiveUsers: stat.Data.DailyActiveUsers,
 		})
 	}