Capture and display daily active user stats

cmd/apiserver/main.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"crypto/sha1"
 	"fmt"
-	"net/http"
 	"os"
 	"strconv"
 
@@ -182,7 +181,7 @@ func realMain(ctx context.Context) error {
 		sub := r.PathPrefix("/api/verify").Subrouter()
 		sub.Use(requireAPIKey)
 		sub.Use(processFirewall)
-		sub.Use(processChaff(verifyChaffTracker))
+		sub.Use(middleware.ProcessChaff(db, verifyChaffTracker))
 		sub.Use(rateLimit)
 
 		// POST /api/verify
@@ -197,7 +196,7 @@ func realMain(ctx context.Context) error {
 		sub := r.PathPrefix("/api/certificate").Subrouter()
 		sub.Use(requireAPIKey)
 		sub.Use(processFirewall)
-		sub.Use(processChaff(certChaffTracker))
+		sub.Use(middleware.ProcessChaff(db, certChaffTracker))
 		sub.Use(rateLimit)
 
 		// POST /api/certificate
@@ -216,14 +215,6 @@ func realMain(ctx context.Context) error {
 	return srv.ServeHTTPHandler(ctx, handlers.CombinedLoggingHandler(os.Stdout, r))
 }
 
-func processChaff(t *chaff.Tracker) mux.MiddlewareFunc {
-	detector := chaff.HeaderDetector("X-Chaff")
-
-	return func(next http.Handler) http.Handler {
-		return t.HandleTrack(detector, next)
-	}
-}
-
 // makePadFromChaff makes a Padding structure from chaff data.
 // Note, the random chaff data will be longer than necessary, so we shorten it.
 func makePadFromChaff(s string) api.Padding {

cmd/server/assets/realmadmin/show.html

@@ -36,6 +36,23 @@ <h1>Realm stats</h1>
       </small>
     </div>
 
+    <div class="card mb-3">
+      <div class="card-header">
+        <span class="oi oi-bar-chart mr-2 ml-n1"></span>
+        Daily active users
+        <span class="font-weight-bold float-right" data-toggle="tooltip"
+          title="These are daily active users as reported by the apps. See the verification server API documentation for how to populate these metrics.">?</span>
+      </div>
+      <div id="daily_active_users_chart" class="container d-flex h-100 w-100" style="min-height:400px;">
+        <p class="justify-content-center align-self-center text-center font-italic w-100">Loading chart...</p>
+      </div>
+      <small class="card-footer text-muted text-right">
+        <span class="mr-1">Export as:</span>
+        <a href="/realm/stats.csv" class="mr-1">CSV</a>
+        <a href="/realm/stats.json">JSON</a>
+      </small>
+    </div>
+
     <div class="row">
       <div class="col-lg-6 pr-2">
         <div class="card mb-3">
@@ -84,6 +101,7 @@ <h1>Realm stats</h1>
   <script src="https://www.gstatic.com/charts/loader.js"></script>
   <script>
     let realmChartDiv = document.getElementById('realm_chart');
+    let dailyUsersChartDiv = document.getElementById('daily_active_users_chart');
     let $perUserTable = $('#per_user_table');
     let $perExternalIssuerTable = $('#per_external_issuer_table');
     let dateFormatter;
@@ -100,9 +118,9 @@ <h1>Realm stats</h1>
     });
 
     function drawCharts() {
-      drawRealmChart();
-      drawUsersChart();
-      drawExternalIssuersChart();
+      drawRealmCharts();
+      drawUsersTable();
+      drawExternalIssuersTable();
     }
 
     // utcDate parses the given RFC-3339 date as a javascript date, then
@@ -113,7 +131,7 @@ <h1>Realm stats</h1>
       return new Date(d.getTime() + offset);
     }
 
-    function drawRealmChart() {
+    function drawRealmCharts() {
       $.ajax({
         url: '/realm/stats.json',
         dataType: 'json',
@@ -123,38 +141,67 @@ <h1>Realm stats</h1>
           return;
         }
 
-        var dataTable = new google.visualization.DataTable();
+        // Code stats
+        {
+          var dataTable = new google.visualization.DataTable();
+          dataTable.addColumn('date', 'Date');
+          dataTable.addColumn('number', 'Issued');
+          dataTable.addColumn('number', 'Claimed');
 
-        dataTable.addColumn('date', 'Date');
-        dataTable.addColumn('number', 'Issued');
-        dataTable.addColumn('number', 'Claimed');
+          data.statistics.reverse().forEach(function(row) {
+            dataTable.addRow([utcDate(row.date), row.data.codes_issued, row.data.codes_claimed]);
+          });
 
-        data.statistics.reverse().forEach(function(row) {
-          dataTable.addRow([utcDate(row.date), row.data.codes_issued, row.data.codes_claimed]);
-        });
+          dateFormatter.format(dataTable, 0);
+
+          let options = {
+            colors: ['#007bff', '#ff7b00'],
+            chartArea: {
+              left: 30, // leave room for y-axis labels
+              width: '100%'
+            },
+            hAxis: { format: 'M/d' },
+            legend: { position: 'top' },
+            width: '100%'
+          };
+
+          let chart = new google.visualization.LineChart(realmChartDiv);
+          chart.draw(dataTable, options);
+        }
+
+        // Daily actives
+        {
+          var dataTable = new google.visualization.DataTable();
+          dataTable.addColumn('date', 'Date');
+          dataTable.addColumn('number', 'Users');
+
+          data.statistics.reverse().forEach(function(row) {
+            dataTable.addRow([utcDate(row.date), row.data.daily_active_users]);
+          });
 
-        dateFormatter.format(dataTable, 0);
+          dateFormatter.format(dataTable, 0);
 
-        let options = {
-          colors: ['#007bff', '#ff7b00'],
-          chartArea: {
-            left: 30, // leave room for y-axis labels
+          let options = {
+            colors: ['#007bff', '#ff7b00'],
+            chartArea: {
+              left: 30, // leave room for y-axis labels
+              width: '100%'
+            },
+            hAxis: { format: 'M/d' },
+            legend: 'none',
             width: '100%'
-          },
-          hAxis: { format: 'M/d' },
-          legend: { position: 'top' },
-          width: '100%'
-        };
-
-        let chart = new google.visualization.LineChart(realmChartDiv);
-        chart.draw(dataTable, options);
+          };
+
+          let chart = new google.visualization.LineChart(dailyUsersChartDiv);
+          chart.draw(dataTable, options);
+        }
       })
       .fail(function(xhr, status, err) {
         flash.error('Failed to render realm stats: ' + err);
       });
     }
 
-    function drawUsersChart() {
+    function drawUsersTable() {
       $.ajax({
         url: '/realm/stats.json',
         data: { scope: 'user' },
@@ -233,7 +280,7 @@ <h1>Realm stats</h1>
       });
     }
 
-    function drawExternalIssuersChart() {
+    function drawExternalIssuersTable() {
       $.ajax({
         url: '/realm/stats.json',
         data: { scope: 'external' },

docs/api.md

@@ -338,6 +338,12 @@ curl https://apiserver.example.com/api/verify \
   --data '{"padding":"base64 encoded padding"}'
 ```
 
+If the `X-Chaff` header has a value of "daily" (case-insensitive), the request
+will be counted toward the realm's daily active user count. Since the server
+records data in UTC time, clients should send this value once **per UTC day** if
+they are collecting server-side adoption metrics. The server will still respond
+with fake data that should be ignored per guidance below.
+
 The client should still send a real request with a real request body (the body
 will not be processed). The server will respond with a fake response that your
 client **MUST NOT** process or parse. The response will not be a valid JSON

pkg/controller/middleware/chaff.go

@@ -0,0 +1,77 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middleware
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-verification-server/internal/project"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller"
+	"github.com/google/exposure-notifications-verification-server/pkg/database"
+	"github.com/gorilla/mux"
+	"github.com/mikehelmick/go-chaff"
+)
+
+const (
+	// ChaffHeader is the chaff header key.
+	ChaffHeader = "X-Chaff"
+
+	// ChaffDailyKey is the key to check if the chaff should be counted toward
+	// daily stats.
+	ChaffDailyKey = "daily"
+)
+
+// ProcessChaff injects the chaff processing middleware. If chaff requests send
+// a value of "daily" (case-insensitive), they will be counted toward the
+// realm's total active users and return a chaff response. Any other values will
+// only return a chaff response.
+//
+// This must come after RequireAPIKey.
+func ProcessChaff(db *database.Database, t *chaff.Tracker) mux.MiddlewareFunc {
+	detector := chaff.HeaderDetector(ChaffHeader)
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx := r.Context()
+			now := time.Now().UTC()
+
+			logger := logging.FromContext(ctx).Named("middleware.ProcessChaff")
+
+			chaffValue := project.TrimSpace(strings.ToLower(r.Header.Get(ChaffHeader)))
+			if chaffValue == ChaffDailyKey {
+				currentRealm := controller.RealmFromContext(ctx)
+				if currentRealm == nil {
+					logger.Error("missing current realm in context")
+				} else {
+					// Increment DAU asynchronously and out-of-band for the request. These
+					// statistics are best-effort and we don't want to block or delay
+					// rendering to populate stats.
+					go func() {
+						if err := currentRealm.IncrementDailyActiveUsers(db, now); err != nil {
+							logger.Errorw("failed to increment daily active stats",
+								"realm", currentRealm.ID,
+								"error", err)
+						}
+					}()
+				}
+			}
+
+			t.HandleTrack(detector, next).ServeHTTP(w, r)
+		})
+	}
+}

pkg/database/migrations.go

@@ -1725,6 +1725,17 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate {
 				return tx.DropTable(&ExternalIssuerStat{}).Error
 			},
 		},
+		{
+			ID: "00071-AddDailyActiveUsersToRealmStats",
+			Migrate: func(tx *gorm.DB) error {
+				sql := `ALTER TABLE realm_stats ADD COLUMN IF NOT EXISTS daily_active_users INTEGER DEFAULT 0`
+				return tx.Exec(sql).Error
+			},
+			Rollback: func(tx *gorm.DB) error {
+				sql := `ALTER TABLE realm_stats DROP COLUMN IF EXISTS daily_active_users`
+				return tx.Exec(sql).Error
+			},
+		},
 	})
 }
 

pkg/database/realm.go

@@ -1267,7 +1267,8 @@ func (r *Realm) Stats(db *Database, start, stop time.Time) (RealmStats, error) {
 			d.date AS date,
 			$1 AS realm_id,
 			COALESCE(s.codes_issued, 0) AS codes_issued,
-			COALESCE(s.codes_claimed, 0) AS codes_claimed
+			COALESCE(s.codes_claimed, 0) AS codes_claimed,
+			COALESCE(s.daily_active_users, 0) AS daily_active_users
 		FROM (
 			SELECT date::date FROM generate_series($2, $3, '1 day'::interval) date
 		) d
@@ -1394,6 +1395,24 @@ func (r *Realm) QuotaKey(hmacKey []byte) (string, error) {
 	return fmt.Sprintf("realm:quota:%s", dig), nil
 }
 
+// IncrementDailyActiveUsers increments the daily active users for the realm by
+// the provided amount.
+func (r *Realm) IncrementDailyActiveUsers(db *Database, now time.Time) error {
+	date := timeutils.Midnight(now)
+
+	sql := `
+		INSERT INTO realm_stats (date, realm_id, daily_active_users)
+			VALUES ($1, $2, 1)
+		ON CONFLICT (date, realm_id) DO UPDATE
+			SET daily_active_users = realm_stats.daily_active_users + 1
+	`
+
+	if err := db.db.Exec(sql, date, r.ID).Error; err != nil {
+		return fmt.Errorf("failed to increment daily active users for realm %d: %w", r.ID, err)
+	}
+	return nil
+}
+
 // ToCIDRList converts the newline-separated and/or comma-separated CIDR list
 // into an array of strings.
 func ToCIDRList(s string) ([]string, error) {