diff --git a/cmd/server/assets/realmadmin/stats.html b/cmd/server/assets/realmadmin/stats.html index 4b88cd2fb..adbeaebab 100644 --- a/cmd/server/assets/realmadmin/stats.html +++ b/cmd/server/assets/realmadmin/stats.html @@ -259,14 +259,6 @@
Codes issued by external issuers by day
drawExternalIssuersTable(); } - // utcDate parses the given RFC-3339 date as a javascript date, then - // converts it to a UTC date. - function utcDate(str) { - let d = new Date(str); - let offset = d.getTimezoneOffset() * 60 * 1000; - return new Date(d.getTime() + offset); - } - function drawRealmCharts() { $.ajax({ url: '/realm/stats.json', diff --git a/cmd/server/assets/static/js/application.js b/cmd/server/assets/static/js/application.js index e8f2e9f46..0a7e9a9fd 100644 --- a/cmd/server/assets/static/js/application.js +++ b/cmd/server/assets/static/js/application.js @@ -694,3 +694,11 @@ function setTimeOrExpired(element, time, expiredCallback) { } return element.html(`${prefix} ${time}`.trim()); } + +// utcDate parses the given RFC-3339 date as a javascript date, then converts it +// to a UTC date. +function utcDate(str) { + let d = new Date(str); + let offset = d.getTimezoneOffset() * 60 * 1000; + return new Date(d.getTime() + offset); +} diff --git a/cmd/server/assets/users/show.html b/cmd/server/assets/users/show.html index cb0ab51e1..0b5862c72 100644 --- a/cmd/server/assets/users/show.html +++ b/cmd/server/assets/users/show.html @@ -17,16 +17,17 @@ {{template "flash" .}}

{{$user.Name}}

-

- Edit -

-

- Here is information about the user. -

+

Here is information about the user.

-
Details
-
+
+ + Details about {{$user.Name}} + + + +
+
Name
{{$user.Name}} @@ -37,6 +38,15 @@
Email
{{$user.Email}}
+
Password
+
+ + Send password reset + +
+
Realm admin
{{if $user.CanAdminRealm $currentRealm.ID}} @@ -45,74 +55,115 @@
Realm admin
Disabled
{{end}}
- - Send password reset
-
Statistics
-
- {{if $stats}} -
- Loading chart... -
- - - - - - - - - {{range $stat := $stats}} - - - - - {{end}} - -
DateKeys issued
{{$stat.Date.Format "2006-01-02"}}{{$stat.CodesIssued}}
-
- This data is refreshed every 5 minutes. +
+ + Statistics +
+
+
+

Loading chart...

- {{else}} -

This user has not recently issued any codes.

- {{end}}
+ + + This data is refreshed every 30 minutes. + Learn more + + + Export as: + CSV + JSON + +
- ← All users + - {{if $stats}} - - + - {{end}} {{end}} diff --git a/internal/routes/server.go b/internal/routes/server.go index 5e3582039..8f8a084f6 100644 --- a/internal/routes/server.go +++ b/internal/routes/server.go @@ -377,6 +377,8 @@ func userRoutes(r *mux.Router, c *user.Controller) { r.Handle("/{id:[0-9]+}", c.HandleShow()).Methods("GET") r.Handle("/{id:[0-9]+}", c.HandleUpdate()).Methods("PATCH") r.Handle("/{id:[0-9]+}", c.HandleDelete()).Methods("DELETE") + r.Handle("/{id:[0-9]+}/stats.json", c.HandleUserStats()).Methods("GET") + r.Handle("/{id:[0-9]+}/stats.csv", c.HandleUserStats()).Methods("GET") r.Handle("/{id:[0-9]+}/reset-password", c.HandleResetPassword()).Methods("POST") } diff --git a/internal/routes/server_test.go b/internal/routes/server_test.go index 791dcf8ab..bd61b480e 100644 --- a/internal/routes/server_test.go +++ b/internal/routes/server_test.go @@ -151,6 +151,12 @@ func TestRoutes_userRoutes(t *testing.T) { { req: httptest.NewRequest("DELETE", "/12345", nil), }, + { + req: httptest.NewRequest("GET", "/12345/stats.json", nil), + }, + { + req: httptest.NewRequest("GET", "/12345/stats.csv", nil), + }, { req: httptest.NewRequest("POST", "/12345/reset-password", nil), }, diff --git a/pkg/controller/user/controller.go b/pkg/controller/user/controller.go index eacaed5d7..58386a0aa 100644 --- a/pkg/controller/user/controller.go +++ b/pkg/controller/user/controller.go @@ -50,3 +50,10 @@ func New( h: h, } } + +func (c *Controller) findUser(currentUser *database.User, realm *database.Realm, id interface{}) (*database.User, error) { + if currentUser.SystemAdmin { + return c.db.FindUser(id) + } + return realm.FindUser(c.db, id) +} diff --git a/pkg/controller/user/create.go b/pkg/controller/user/create.go index ea7fed899..0cd201361 100644 --- a/pkg/controller/user/create.go +++ b/pkg/controller/user/create.go @@ -104,13 +104,7 @@ func (c *Controller) HandleCreate() http.Handler { flash.Alert("Successfully created user %v.", user.Name) - stats, err := c.getStats(ctx, user, realm) - if err != nil { - controller.InternalError(w, r, c.h, err) - return - } - - c.renderShow(ctx, w, user, stats) + c.renderShow(ctx, w, user) }) } diff --git a/pkg/controller/user/show.go b/pkg/controller/user/show.go index d7b8a7760..2603baf3c 100644 --- a/pkg/controller/user/show.go +++ b/pkg/controller/user/show.go @@ -16,11 +16,8 @@ package user import ( "context" - "fmt" "net/http" - "time" - "github.com/google/exposure-notifications-verification-server/pkg/cache" "github.com/google/exposure-notifications-verification-server/pkg/controller" "github.com/google/exposure-notifications-verification-server/pkg/database" "github.com/gorilla/mux" @@ -66,43 +63,12 @@ func (c *Controller) Show(w http.ResponseWriter, r *http.Request, resetPassword return } - userStats, err := c.getStats(ctx, user, realm) - if err != nil { - controller.InternalError(w, r, c.h, err) - return - } - - c.renderShow(ctx, w, user, userStats) -} - -// Get and cache the stats for this user. -func (c *Controller) getStats(ctx context.Context, user *database.User, realm *database.Realm) ([]*database.UserStats, error) { - var stats []*database.UserStats - cacheKey := &cache.Key{ - Namespace: "stats:user", - Key: fmt.Sprintf("%d:%d", realm.ID, user.ID), - } - if err := c.cacher.Fetch(ctx, cacheKey, &stats, 5*time.Minute, func() (interface{}, error) { - now := time.Now().UTC() - past := now.Add(-14 * 24 * time.Hour) - return user.Stats(c.db, realm.ID, past, now) - }); err != nil { - return nil, err - } - return stats, nil -} - -func (c *Controller) findUser(currentUser *database.User, realm *database.Realm, id interface{}) (*database.User, error) { - if currentUser.SystemAdmin { - return c.db.FindUser(id) - } - return realm.FindUser(c.db, id) + c.renderShow(ctx, w, user) } -func (c *Controller) renderShow(ctx context.Context, w http.ResponseWriter, user *database.User, stats []*database.UserStats) { +func (c *Controller) renderShow(ctx context.Context, w http.ResponseWriter, user *database.User) { m := controller.TemplateMapFromContext(ctx) m.Title("User: %s", user.Name) m["user"] = user - m["stats"] = stats c.h.RenderHTML(w, "users/show", m) } diff --git a/pkg/controller/user/stats.go b/pkg/controller/user/stats.go new file mode 100644 index 000000000..304b5df51 --- /dev/null +++ b/pkg/controller/user/stats.go @@ -0,0 +1,106 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package user + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/google/exposure-notifications-verification-server/internal/project" + "github.com/google/exposure-notifications-verification-server/pkg/cache" + "github.com/google/exposure-notifications-verification-server/pkg/controller" + "github.com/google/exposure-notifications-verification-server/pkg/database" + "github.com/gorilla/mux" +) + +const statsCacheTimeout = 30 * time.Minute + +func (c *Controller) HandleUserStats() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + + session := controller.SessionFromContext(ctx) + if session == nil { + controller.MissingSession(w, r, c.h) + return + } + + realm := controller.RealmFromContext(ctx) + if realm == nil { + controller.MissingRealm(w, r, c.h) + return + } + + currentUser := controller.UserFromContext(ctx) + if currentUser == nil { + controller.MissingUser(w, r, c.h) + return + } + + // Pull the user from the id. + user, err := c.findUser(currentUser, realm, vars["id"]) + if err != nil { + if database.IsNotFound(err) { + controller.Unauthorized(w, r, c.h) + return + } + + controller.InternalError(w, r, c.h, err) + return + } + + stats, err := c.getStats(ctx, user, realm) + if err != nil { + controller.InternalError(w, r, c.h, err) + return + } + + pth := r.URL.Path + switch { + case strings.HasSuffix(pth, ".csv"): + nowFormatted := time.Now().UTC().Format(project.RFC3339Squish) + filename := fmt.Sprintf("%s-user-stats.csv", nowFormatted) + c.h.RenderCSV(w, http.StatusOK, filename, stats) + return + case strings.HasSuffix(pth, ".json"): + c.h.RenderJSON(w, http.StatusOK, stats) + return + default: + controller.InternalError(w, r, c.h, fmt.Errorf("unknown path %q", pth)) + return + } + }) +} + +func (c *Controller) getStats(ctx context.Context, user *database.User, realm *database.Realm) (database.UserStats, error) { + now := time.Now().UTC() + past := now.Add(-30 * 24 * time.Hour) + + var stats database.UserStats + cacheKey := &cache.Key{ + Namespace: "stats:user", + Key: fmt.Sprintf("%d:%d", realm.ID, user.ID), + } + if err := c.cacher.Fetch(ctx, cacheKey, &stats, statsCacheTimeout, func() (interface{}, error) { + return user.Stats(c.db, realm.ID, past, now) + }); err != nil { + return nil, err + } + return stats, nil +} diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index 915eeb1f5..b5151e1be 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -415,7 +415,7 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate { ID: "00017-AddIssuerIDColumns", Migrate: func(tx *gorm.DB) error { logger.Debugw("adding issuer id columns to verification codes") - err := tx.AutoMigrate(&VerificationCode{}, &UserStats{}, &AuthorizedAppStats{}).Error + err := tx.AutoMigrate(&VerificationCode{}, &UserStat{}, &AuthorizedAppStats{}).Error return err }, @@ -430,7 +430,7 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate { return err } } - if err := tx.DropTableIfExists(&UserStats{}).Error; err != nil { + if err := tx.DropTableIfExists(&UserStat{}).Error; err != nil { return err } return nil diff --git a/pkg/database/realm_test.go b/pkg/database/realm_test.go index dfd860160..c3ff9fdf8 100644 --- a/pkg/database/realm_test.go +++ b/pkg/database/realm_test.go @@ -78,13 +78,13 @@ func TestPerUserRealmStats(t *testing.T) { // Add some stats per user. for i := 0; i < numDays; i++ { - stat := &UserStats{ + stat := &UserStat{ RealmID: realm.ID, UserID: user.ID, Date: startDate.Add(time.Duration(i) * 24 * time.Hour), CodesIssued: uint(10 + i + userIdx), } - if err := db.SaveUserStats(stat); err != nil { + if err := db.SaveUserStat(stat); err != nil { t.Fatalf("error saving user stats %v", err) } } diff --git a/pkg/database/user.go b/pkg/database/user.go index 92c57e870..d1b5b66e1 100644 --- a/pkg/database/user.go +++ b/pkg/database/user.go @@ -206,26 +206,36 @@ func (db *Database) FindUserByEmail(email string) (*User, error) { // Stats returns the usage statistics for this user at the provided realm. If no // stats exist, it returns an empty array. -func (u *User) Stats(db *Database, realmID uint, start, stop time.Time) ([]*UserStats, error) { - var stats []*UserStats +func (u *User) Stats(db *Database, realmID uint, start, stop time.Time) (UserStats, error) { + start = timeutils.UTCMidnight(start) + stop = timeutils.UTCMidnight(stop) - start = timeutils.Midnight(start) - stop = timeutils.Midnight(stop) + if start.After(stop) { + return nil, ErrBadDateRange + } - if err := db.db. - Model(&UserStats{}). - Where("user_id = ?", u.ID). - Where("realm_id = ?", realmID). - Where("date >= ? AND date <= ?", start, stop). - Order("date DESC"). - Find(&stats). - Error; err != nil { + // Pull the stats by generating the full date range, then join on stats. This + // will ensure we have a full list (with values of 0 where appropriate) to + // ensure continuity in graphs. + sql := ` + SELECT + d.date AS date, + $1 AS user_id, + $2 AS realm_id, + COALESCE(s.codes_issued, 0) AS codes_issued + FROM ( + SELECT date::date FROM generate_series($3, $4, '1 day'::interval) date + ) d + LEFT JOIN user_stats s ON s.user_id = $1 AND s.realm_id = $2 AND s.date = d.date + ORDER BY date DESC` + + var stats []*UserStat + if err := db.db.Raw(sql, u.ID, realmID, start, stop).Scan(&stats).Error; err != nil { if IsNotFound(err) { return stats, nil } return nil, err } - return stats, nil } diff --git a/pkg/database/user_stats.go b/pkg/database/user_stats.go index 47dfafce6..2780c6046 100644 --- a/pkg/database/user_stats.go +++ b/pkg/database/user_stats.go @@ -15,22 +15,139 @@ package database import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" "time" + "github.com/google/exposure-notifications-verification-server/internal/icsv" + "github.com/google/exposure-notifications-verification-server/internal/project" "github.com/jinzhu/gorm" ) -// UserStats represents statistics related to a user in the database. -type UserStats struct { - Date time.Time `gorm:"date;"` - UserID uint `gorm:"user_id;"` - RealmID uint `gorm:"realm_id;"` - CodesIssued uint `gorm:"codes_issued;"` +var _ icsv.Marshaler = (UserStats)(nil) + +// UserStats represents a logical collection of stats for a user. +type UserStats []*UserStat + +// UserStat represents a single-date statistic for a user. +type UserStat struct { + Date time.Time `gorm:"date; not null;"` + UserID uint `gorm:"user_id; not null;"` + RealmID uint `gorm:"realm_id; default:0;"` + CodesIssued uint `gorm:"codes_issued; default:0;"` +} + +// MarshalCSV returns bytes in CSV format. +func (s UserStats) MarshalCSV() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return nil, nil + } + + var b bytes.Buffer + w := csv.NewWriter(&b) + + if err := w.Write([]string{"date", "user_id", "realm_id", "codes_issued"}); err != nil { + return nil, fmt.Errorf("failed to write CSV header: %w", err) + } + + for i, stat := range s { + if err := w.Write([]string{ + stat.Date.Format(project.RFC3339Date), + strconv.FormatUint(uint64(stat.UserID), 10), + strconv.FormatUint(uint64(stat.RealmID), 10), + strconv.FormatUint(uint64(stat.CodesIssued), 10), + }); err != nil { + return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) + } + } + + w.Flush() + if err := w.Error(); err != nil { + return nil, fmt.Errorf("failed to create CSV: %w", err) + } + + return b.Bytes(), nil +} + +type jsonUserStat struct { + UserID uint `json:"user_id"` + RealmID uint `json:"realm_id"` + Stats []*jsonUserStatStats `json:"statistics"` +} + +type jsonUserStatStats struct { + Date time.Time `json:"date"` + Data *jsonUserStatStatsData `json:"data"` +} + +type jsonUserStatStatsData struct { + CodesIssued uint `json:"codes_issued"` +} + +// MarshalJSON is a custom JSON marshaller. +func (s UserStats) MarshalJSON() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return json.Marshal(struct{}{}) + } + + var stats []*jsonUserStatStats + for _, stat := range s { + stats = append(stats, &jsonUserStatStats{ + Date: stat.Date, + Data: &jsonUserStatStatsData{ + CodesIssued: stat.CodesIssued, + }, + }) + } + + // Sort in descending order. + sort.Slice(stats, func(i, j int) bool { + return stats[i].Date.After(stats[j].Date) + }) + + var result jsonUserStat + result.UserID = s[0].UserID + result.RealmID = s[0].RealmID + result.Stats = stats + + b, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %w", err) + } + return b, nil +} + +func (s *UserStats) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + var result jsonUserStat + if err := json.Unmarshal(b, &result); err != nil { + return err + } + + for _, stat := range result.Stats { + *s = append(*s, &UserStat{ + Date: stat.Date, + UserID: result.UserID, + RealmID: result.RealmID, + CodesIssued: stat.Data.CodesIssued, + }) + } + + return nil } -// SaveUserStats saves some UserStats to the database. -// This function is provided for testing only. -func (db *Database) SaveUserStats(u *UserStats) error { +// SaveUserStat saves some UserStats to the database. This function is provided +// for testing only. +func (db *Database) SaveUserStat(u *UserStat) error { return db.db.Transaction(func(tx *gorm.DB) error { if err := tx.Save(u).Error; err != nil { return err diff --git a/pkg/database/vercode_test.go b/pkg/database/vercode_test.go index 2db93cb89..22022f3f4 100644 --- a/pkg/database/vercode_test.go +++ b/pkg/database/vercode_test.go @@ -455,7 +455,7 @@ func TestStatDatesOnCreate(t *testing.T) { { var stats []*RealmUserStat if err := db.db. - Model(&UserStats{}). + Model(&UserStat{}). Select("*"). Scan(&stats). Error; err != nil { @@ -477,7 +477,7 @@ func TestStatDatesOnCreate(t *testing.T) { if len(test.code.IssuingExternalID) != 0 { var stats []*ExternalIssuerStat if err := db.db. - Model(&ExternalIssuerStats{}). + Model(&ExternalIssuerStat{}). Select("*"). Scan(&stats). Error; err != nil {