Sync android apps from json file

cmd/app-sync/main.go

@@ -106,7 +106,7 @@ func realMain(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to create cleanup controller: %w", err)
 	}
-	r.Handle("/", appSyncController.HandleSync(ctx)).Methods("GET")
+	r.Handle("/", appSyncController.HandleSync()).Methods("GET")
 
 	srv, err := server.New(cfg.Port)
 	if err != nil {

cmd/e2e-runner/main.go

@@ -17,8 +17,6 @@ package main
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/sha256"
 	"fmt"
 	"net/http"
 	"os"
@@ -29,6 +27,7 @@ import (
 	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 
+	"github.com/google/exposure-notifications-verification-server/internal/project"
 	"github.com/google/exposure-notifications-verification-server/pkg/buildinfo"
 	"github.com/google/exposure-notifications-verification-server/pkg/clients"
 	"github.com/google/exposure-notifications-verification-server/pkg/config"
@@ -67,15 +66,6 @@ func main() {
 	logger.Info("successful shutdown")
 }
 
-// Generate random string of 32 characters in length
-func randomString() (string, error) {
-	b := make([]byte, 512)
-	if _, err := rand.Read(b[:]); err != nil {
-		return "", fmt.Errorf("failed to generate random: %v", err)
-	}
-	return fmt.Sprintf("%x", sha256.Sum256(b[:])), nil
-}
-
 func realMain(ctx context.Context) error {
 	logger := logging.FromContext(ctx)
 
@@ -126,7 +116,7 @@ func realMain(ctx context.Context) error {
 	}
 
 	// Create new API keys
-	suffix, err := randomString()
+	suffix, err := project.RandomString()
 	if err != nil {
 		return fmt.Errorf("failed to create suffix string for API keys: %w", err)
 	}

internal/project/random.go

@@ -0,0 +1,41 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package project defines global project helpers.
+package project
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+)
+
+// RandomString generates a random string of 32 characters in length
+func RandomString() (string, error) {
+	b := make([]byte, 512)
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("failed to generate random: %w", err)
+	}
+	return fmt.Sprintf("%x", sha256.Sum256(b[:])), nil
+}
+
+// RandomBase64String encodes a random base64 string of a given length.
+func RandomBase64String(len int) (string, error) {
+	b := make([]byte, len)
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("failed to generate random: %w", err)
+	}
+	return base64.URLEncoding.EncodeToString(b), nil
+}

pkg/api/api.go

@@ -24,7 +24,7 @@ import (
 )
 
 const (
-	// TestTypeConfirmed is the string that represents a confirmed covid-19 test.
+	// TestTypeConfirmed is the string that represents a confirmed COVID-19 test.
 	TestTypeConfirmed = "confirmed"
 	// TestTypeLikely is the string that represents a clinical diagnosis.
 	TestTypeLikely = "likely"
@@ -56,6 +56,7 @@ const (
 	ErrMissingDate = "missing_date"
 
 	// Certificate API responses
+
 	// ErrTokenInvalid indicates the token provided is unknown or already used
 	ErrTokenInvalid = "token_invalid"
 	// ErrTokenExpired indicates that the token provided is known but expired.
@@ -272,7 +273,7 @@ type ExpireCodeResponse struct {
 	// After this time the code will no longer be accepted and is eligible for deletion.
 	ExpiresAtTimestamp int64 `json:"expiresAtTimestamp"`
 
-	// LongExpiresAtTimestamp repesents the time when the long code expires, in
+	// LongExpiresAtTimestamp represents the time when the long code expires, in
 	// UTC seconds since epoch.
 	LongExpiresAtTimestamp int64 `json:"longExpiresAtTimestamp,omitempty"`
 

pkg/clients/apis.go

@@ -17,6 +17,9 @@ package clients
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
+	"io"
 	"net/http"
 	"time"
 
@@ -106,3 +109,24 @@ func GetCertificate(ctx context.Context, hostname, apikey, token, hmac string, t
 	}
 	return &request, &response, nil
 }
+
+// AppSync makes the API call to synchronize mobile apps.
+func AppSync(url string, timeout time.Duration, sizeLimit int64) (*AppsResponse, error) {
+	if url == "" {
+		return nil, errors.New("no APP_SYNC_URL configured")
+	}
+
+	client := http.Client{Timeout: timeout}
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	var apps AppsResponse
+	defer resp.Body.Close()
+
+	if err := json.NewDecoder(io.LimitReader(resp.Body, sizeLimit)).Decode(&apps); err != nil {
+		return nil, err
+	}
+	return &apps, nil
+}

pkg/clients/responses.go

@@ -0,0 +1,34 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clients
+
+// AppsResponse is the body for the published list of android apps.
+type AppsResponse struct {
+	Apps []App `json:"apps"`
+}
+
+// App represents single app for the AppResponse body.
+type App struct {
+	Region        string `json:"region"`
+	IsEnx         bool   `json:"is_enx,omitempty"`
+	AndroidTarget `json:"android_target"`
+}
+
+// AndroidTarget holds the android metadata for an App of AppResponse.
+type AndroidTarget struct {
+	Namespace              string `json:"namespace"`
+	PackageName            string `json:"package_name"`
+	SHA256CertFingerprints string `json:"sha256_cert_fingerprints"`
+}

pkg/config/appsync_server_config.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"time"
 
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 
@@ -41,7 +42,9 @@ type AppSyncConfig struct {
 	RateLimit uint64 `env:"RATE_LIMIT,default=60"`
 
 	// AppSync config
-	AppSyncURL string `env:"APP_SYNC_URL, default=https://www.gstatic.com/exposurenotifications/apps.json"`
+	AppSyncURL         string        `env:"APP_SYNC_URL"`
+	FileSizeLimitBytes int64         `env:"APP_SYNC_SIZE_LIMIT, default=10240"`
+	Timeout            time.Duration `env:"APP_SYNC_TIMEOUT, default=1m"`
 }
 
 // NewAppSyncConfig returns the environment config for the appsync server.
@@ -55,10 +58,14 @@ func NewAppSyncConfig(ctx context.Context) (*AppSyncConfig, error) {
 }
 
 func (c *AppSyncConfig) Validate() error {
+	if c.AppSyncURL == "" {
+		return nil
+	}
+
 	if url, err := url.Parse(c.AppSyncURL); err != nil {
 		return fmt.Errorf("unable to parse APP_SYNC_URL: %v", err)
 	} else if url == nil {
-		return errors.New("expecting a value for APP_SYNC_URL")
+		return errors.New("expecting a URL value for APP_SYNC_URL")
 	}
 
 	return nil

pkg/controller/appsync/appsync.go

@@ -12,40 +12,152 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package appsync syncs the published list of mobile apps to this server's db.
 package appsync
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"net/http"
+	"net/url"
 
-	"github.com/google/exposure-notifications-verification-server/pkg/config"
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-verification-server/internal/project"
+	"github.com/google/exposure-notifications-verification-server/pkg/clients"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller"
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
-	"github.com/google/exposure-notifications-verification-server/pkg/render"
+	"github.com/hashicorp/go-multierror"
 )
 
-// Controller is a controller for the appsync service.
-type Controller struct {
-	config *config.AppSyncConfig
-	db     *database.Database
-	h      *render.Renderer
-}
-
-// New creates a new appsync controller.
-func New(config *config.AppSyncConfig, db *database.Database, h *render.Renderer) (*Controller, error) {
-	return &Controller{
-		config: config,
-		db:     db,
-		h:      h,
-	}, nil
-}
+const playStoreHost = `play.google.com/store/apps/details`
 
 // HandleSync performs the logic to sync mobile apps.
-func (c *Controller) HandleSync(ctx context.Context) http.Handler {
+func (c *Controller) HandleSync() http.Handler {
+	type AppSyncResult struct {
+		OK     bool    `json:"ok"`
+		Errors []error `json:"errors,omitempty"`
+	}
+
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// TODO(whaught): implement this
-		controller.InternalError(w, r, c.h, errors.New("not implemented"))
+		ctx := r.Context()
+		apps, err := clients.AppSync(c.config.AppSyncURL, c.config.Timeout, c.config.FileSizeLimitBytes)
+		if err != nil {
+			controller.InternalError(w, r, c.h, err)
+			return
+		}
+
+		// If there are any errors, return them
+		if merr := c.syncApps(ctx, apps); merr != nil {
+			if errs := merr.WrappedErrors(); len(errs) > 0 {
+				c.h.RenderJSON(w, http.StatusInternalServerError, &AppSyncResult{
+					OK:     false,
+					Errors: errs,
+				})
+				return
+			}
+		}
+		c.h.RenderJSON(w, http.StatusOK, &AppSyncResult{OK: true})
 	})
 }
+
+// syncApps looks up the realm and associated list of MobileApps for each entry of AppsResponse. Then it
+// checks to see if there exists an app with the AppResponse SHA hash, if not it creates a new MobileApp.
+func (c *Controller) syncApps(ctx context.Context, apps *clients.AppsResponse) *multierror.Error {
+	logger := logging.FromContext(ctx).Named("appsync.syncApps")
+	var merr *multierror.Error
+
+	realms := map[string]*database.Realm{}
+	appsByRealm := map[uint][]*database.MobileApp{}
+
+	for _, app := range apps.Apps {
+
+		realm, err := c.findRealmForApp(app, realms)
+		if err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("unable to lookup realm for region %q: %w", app.Region, err))
+			continue
+		}
+
+		realmApps, err := c.findAppsForRealm(realm.ID, appsByRealm)
+		if err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("unable to list apps for realm %d: %w", realm.ID, err))
+			continue
+		}
+
+		// Find out if this realm's applist already has an app with this fingerprint.
+		hasSHA, hasGeneratedName := false, false
+		for _, a := range realmApps {
+			if a.SHA == app.SHA256CertFingerprints {
+				hasSHA = true
+			}
+			if a.Name == generateAppName(app) {
+				hasGeneratedName = true
+			}
+		}
+
+		// Didn't find an app. make one.
+		if !hasSHA {
+			logger.Infow("app not found during sync, adding", "app", app)
+
+			name := generateAppName(app)
+			if hasGeneratedName { // add a random string to names on collision
+				s, err := project.RandomBase64String(8)
+				if err != nil {
+					merr = multierror.Append(merr, fmt.Errorf("error generating app name: %w", err))
+					continue
+				}
+				name += " " + s
+			}
+
+			var playStoreURL = &url.URL{
+				Scheme:   "https",
+				Host:     playStoreHost,
+				RawQuery: "id=" + app.PackageName,
+			}
+
+			newApp := &database.MobileApp{
+				Name:    name,
+				RealmID: realm.ID,
+				URL:     playStoreURL.String(),
+				OS:      database.OSTypeAndroid,
+				SHA:     app.SHA256CertFingerprints,
+				AppID:   app.PackageName,
+			}
+			if err := c.db.SaveMobileApp(newApp, database.System); err != nil {
+				merr = multierror.Append(merr, fmt.Errorf("failed saving mobile app: %w", err))
+				continue
+			}
+		}
+	}
+	return merr
+}
+
+func (c *Controller) findRealmForApp(
+	app clients.App, realms map[string]*database.Realm) (*database.Realm, error) {
+	var err error
+	realm, has := realms[app.Region]
+	if !has { // Find this apps region and cache it in our realms map
+		realm, err = c.db.FindRealmByRegion(app.Region)
+		if err != nil {
+			return nil, err
+		}
+		realms[app.Region] = realm
+	}
+	return realm, nil
+}
+
+func (c *Controller) findAppsForRealm(
+	realmID uint, appsByRealm map[uint][]*database.MobileApp) ([]*database.MobileApp, error) {
+	var err error
+	realmApps, has := appsByRealm[realmID]
+	if !has { // Find all of the apps for this realm and cache that list in our appByRealmMap
+		realmApps, err = c.db.ListActiveApps(realmID, database.WithAppOS(database.OSTypeAndroid))
+		if err != nil {
+			return nil, err
+		}
+		appsByRealm[realmID] = realmApps
+	}
+	return realmApps, nil
+}
+
+func generateAppName(app clients.App) string {
+	return app.Region + " Android App"
+}