implement clientview fetch

cmd/diffs/main.go

@@ -136,10 +136,11 @@ type gsp func() (spec.Spec, error)
 func serve(parent *kingpin.Application, sps *string, errs io.Writer) {
 	kc := parent.Command("serve", "Starts a local diff-server.")
 	port := kc.Flag("port", "The port to run on").Default("7001").Int()
+	overrideClientViewURL := parent.Flag("clientview", "URL to always use for client view eg 'https://example.com/clientview'").Default("").String()
 	kc.Action(func(_ *kingpin.ParseContext) error {
 		ps := fmt.Sprintf(":%d", *port)
 		log.Printf("Listening on %s...", ps)
-		s := servepkg.NewService(*sps, accounts.Accounts())
+		s := servepkg.NewService(*sps, accounts.Accounts(), *overrideClientViewURL)
 		http.Handle("/", s)
 		return http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)
 	})

cmd/diffs/main_test.go

@@ -77,7 +77,7 @@ func TestServe(t *testing.T) {
 		expectedResponse string
 		expectedError    string
 	}{
-		{"handlePullRequest", `{"baseStateID": "00000000000000000000000000000000", "checksum": "00000000"}`,
+		{"handlePullRequest", `{"baseStateID": "00000000000000000000000000000000", "checksum": "00000000", "clientID": "clientid"}`,
 			`{"stateID":"unhmo677duk3vbjpu0f01eusdep2k7ei","patch":[{"op":"remove","path":"/"}],"checksum":"00000000"}`, ""},
 	}
 

db/handle_pull.go → db/diff.go

@@ -23,7 +23,7 @@ func fullSync(db *DB, from hash.Hash) ([]kv.Operation, Commit) {
 	return r, makeCommit(db.Noms(), types.Ref{}, datetime.Epoch, db.noms.WriteValue(m.NomsMap()), types.String(m.Checksum().String()))
 }
 
-func (db *DB) HandlePull(from hash.Hash, fromChecksum kv.Checksum) ([]kv.Operation, error) {
+func (db *DB) Diff(from hash.Hash, fromChecksum kv.Checksum) ([]kv.Operation, error) {
 	r := []kv.Operation{}
 	v := db.Noms().ReadValue(from)
 	var fc Commit

db/handle_pull_test.go → db/diff_test.go

@@ -11,7 +11,7 @@ import (
 	"roci.dev/diff-server/kv"
 )
 
-func TestHandlePull(t *testing.T) {
+func TestDiff(t *testing.T) {
 	assert := assert.New(t)
 	db, dir := LoadTempDB(assert)
 	fmt.Println(dir)
@@ -208,7 +208,7 @@ func TestHandlePull(t *testing.T) {
 		t.f()
 		c, err := kv.ChecksumFromString(fromChecksum)
 		assert.NoError(err)
-		r, err := db.HandlePull(fromID, *c)
+		r, err := db.Diff(fromID, *c)
 		if t.expectedError == "" {
 			assert.NoError(err, t.label)
 			expected, err := json.Marshal(t.expectedDiff)

kv/map.go

@@ -115,7 +115,7 @@ func (me MapEditor) Get(key string) ([]byte, error) {
 
 // Set sets the value for a given key.
 func (me *MapEditor) Set(key string, value []byte) error {
-	nv, err := nomsjson.FromJSON(bytes.NewReader(value), me.noms, nomsjson.FromOptions{})
+	nv, err := nomsjson.FromJSON(bytes.NewReader(value), me.noms)
 	if err != nil {
 		return err
 	}

serve/accounts/accounts.go

@@ -10,16 +10,19 @@ var (
 			ID:     "sandbox",
 			Name:   "Sandbox",
 			Pubkey: nil,
+			// ClientViewURL: "<url>",
 		},
 		serve.Account{
 			ID:     "1",
 			Name:   "Rocicorp",
 			Pubkey: []byte("-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4DwXA3SHZ7TpzahAgOTRRgblBGxL\ndOHVmZ/J1bgBuuxMZzkassAsUCFCaMNu5HZuFUh98kA1laxZzs78O9EDQw==\n-----END PUBLIC KEY-----"),
+			// ClientViewURL: "<url>",
 		},
 		serve.Account{
 			ID:     "2",
 			Name:   "Turtle Technologies, Inc.",
 			Pubkey: []byte("-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEwNhpc2KRnxQRq2YETuKJShSC623E\nFFlAf4j2fFDKkEDM0Py9HCd3UDmxLyF7HUv9GC9mA+78HahKQnF8F37jLQ==\n-----END PUBLIC KEY-----"),
+			// ClientViewURL: "<url>",
 		},
 	}
 )

serve/client_view.go

@@ -0,0 +1,47 @@
+package serve
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	servetypes "roci.dev/diff-server/serve/types"
+)
+
+type ClientViewGetter struct {
+	url string
+}
+
+// Get fetches a client view. It returns an error if the response from the data layer doesn't have
+// a lastTransactionID.
+func (g ClientViewGetter) Get(req servetypes.ClientViewRequest, authToken string) (servetypes.ClientViewResponse, error) {
+	reqBody, err := json.Marshal(req)
+	if err != nil {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("could not marshal ClientViewRequest: %w", err)
+	}
+	httpReq, err := http.NewRequest("POST", g.url, bytes.NewReader(reqBody))
+	if err != nil {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("could not create client view http request: %w", err)
+	}
+	httpReq.Header.Add("Authorization", authToken)
+	httpResp, err := http.DefaultClient.Do(httpReq)
+	if err != nil {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("error sending client view http request: %w", err)
+	}
+	if httpResp.StatusCode != http.StatusOK {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("client view fetch http request returned %s", httpResp.Status)
+	}
+	var resp servetypes.ClientViewResponse
+	var r io.Reader = httpResp.Body
+	defer httpResp.Body.Close()
+	err = json.NewDecoder(r).Decode(&resp)
+	if err != nil {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("couldnt decode client view response: %w", err)
+	}
+	if resp.LastTransactionID == "" {
+		return servetypes.ClientViewResponse{}, fmt.Errorf("malformed response %v missing lastTransactionID", resp)
+	}
+	return resp, nil
+}

serve/client_view_test.go

@@ -0,0 +1,84 @@
+package serve
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	servetypes "roci.dev/diff-server/serve/types"
+)
+
+func TestClientViewGetter_Get(t *testing.T) {
+	assert := assert.New(t)
+
+	type args struct {
+	}
+	tests := []struct {
+		name      string
+		req       servetypes.ClientViewRequest
+		authToken string
+		respCode  int
+		respBody  string
+		want      servetypes.ClientViewResponse
+		wantErr   string
+	}{
+		{
+			"ok",
+			servetypes.ClientViewRequest{ClientID: "clientid"},
+			"authtoken",
+			http.StatusOK,
+			`{"clientView": "clientview", "lastTransactionID": "ltid"}`,
+			servetypes.ClientViewResponse{ClientView: []byte("\"clientview\""), LastTransactionID: "ltid"},
+			"",
+		},
+		{
+			"error",
+			servetypes.ClientViewRequest{ClientID: "clientid"},
+			"authtoken",
+			http.StatusBadRequest,
+			``,
+			servetypes.ClientViewResponse{},
+			"400",
+		},
+		{
+			"missing last transaction id",
+			servetypes.ClientViewRequest{ClientID: "clientid"},
+			"authtoken",
+			http.StatusOK,
+			`{"clientView": "foo", "lastTransactionID": ""}`,
+			servetypes.ClientViewResponse{},
+			"lastTransactionID",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				var reqBody servetypes.ClientViewRequest
+				err := json.NewDecoder(r.Body).Decode(&reqBody)
+				assert.NoError(err, tt.name)
+				assert.Equal(tt.req.ClientID, reqBody.ClientID, tt.name)
+				assert.Equal(tt.authToken, r.Header.Get("Authorization"), tt.name)
+				w.WriteHeader(tt.respCode)
+				w.Write([]byte(tt.respBody))
+			}))
+
+			g := ClientViewGetter{
+				url: server.URL,
+			}
+			got, err := g.Get(tt.req, tt.authToken)
+			if tt.wantErr == "" {
+				assert.NoError(err)
+			} else {
+				assert.Error(err)
+				assert.Regexp(tt.wantErr, err.Error(), tt.name)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ClientViewGetter.Get() case %s got %v (clientview=%s), want %v (clientview=%s)", tt.name, got, string(got.ClientView), tt.want, string(tt.want.ClientView))
+			}
+		})
+	}
+}

serve/prod/prod.go

@@ -21,7 +21,7 @@ const (
 )
 
 var (
-	svc = serve.NewService("aws:replicant/aa-replicant2", accounts.Accounts())
+	svc = serve.NewService("aws:replicant/aa-replicant2", accounts.Accounts(), "")
 )
 
 func init() {

serve/server.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -25,23 +26,31 @@ import (
 	"roci.dev/diff-server/db"
 	"roci.dev/diff-server/kv"
 	servetypes "roci.dev/diff-server/serve/types"
+	nomsjson "roci.dev/diff-server/util/noms/json"
 )
 
+type clientViewGetter interface {
+	Get(req servetypes.ClientViewRequest, authToken string) (servetypes.ClientViewResponse, error)
+}
+
 // server is a single Replicant instance. The Replicant service runs many such instances.
 type server struct {
 	router *httprouter.Router
 	db     *db.DB
+	cvg    clientViewGetter
 	mu     sync.Mutex
 }
 
-func newServer(cs chunks.ChunkStore, urlPrefix string) (*server, error) {
+// cvg may be nil, in which case the server skips the client view request in pull, which is
+// useful if you are populating the db directly or in tests.
+func newServer(cs chunks.ChunkStore, urlPrefix string, cvg clientViewGetter) (*server, error) {
 	router := httprouter.New()
 	noms := datas.NewDatabase(cs)
 	db, err := db.New(noms)
 	if err != nil {
 		return nil, err
 	}
-	s := &server{router: router, db: db}
+	s := &server{router: router, db: db, cvg: cvg}
 	s.router.POST(fmt.Sprintf("%s/handlePullRequest", urlPrefix), func(rw http.ResponseWriter, req *http.Request, ps httprouter.Params) {
 		body := bytes.Buffer{}
 		_, err := io.Copy(&body, req.Body)
@@ -50,23 +59,40 @@ func newServer(cs chunks.ChunkStore, urlPrefix string) (*server, error) {
 			serverError(rw, err)
 			return
 		}
-		var hsreq servetypes.PullRequest
-		err = json.Unmarshal(body.Bytes(), &hsreq)
+		var preq servetypes.PullRequest
+		err = json.Unmarshal(body.Bytes(), &preq)
 		if err != nil {
 			serverError(rw, err)
 			return
 		}
-
-		from, ok := hash.MaybeParse(hsreq.BaseStateID)
+		from, ok := hash.MaybeParse(preq.BaseStateID)
 		if !ok {
 			clientError(rw, 400, "Invalid baseStateID")
 			return
 		}
-		fromChecksum, err := kv.ChecksumFromString(hsreq.Checksum)
+		fromChecksum, err := kv.ChecksumFromString(preq.Checksum)
 		if err != nil {
 			clientError(rw, 400, "Invalid checksum")
 		}
-		patch, err := s.db.HandlePull(from, *fromChecksum)
+		if preq.ClientID == "" {
+			clientError(rw, 400, "Missing ClientID")
+			return
+		}
+		if cvg == nil {
+			log.Print("WARNING: not fetching new client view: no url provided via account or --clientview")
+		} else {
+			var cvgError error
+			cvreq := servetypes.ClientViewRequest{ClientID: preq.ClientID}
+			cvresp, cvgError := cvg.Get(cvreq, "") // TODO fritz pass auth token along
+			if cvgError == nil {
+				cvgError = storeNewClientView(db, cvresp.ClientView)
+			}
+			if cvgError != nil {
+				log.Printf("WARNING: got error fetching clientview: %s", cvgError)
+			}
+		}
+
+		patch, err := s.db.Diff(from, *fromChecksum)
 		if err != nil {
 			serverError(rw, err)
 			return
@@ -102,6 +128,24 @@ func newServer(cs chunks.ChunkStore, urlPrefix string) (*server, error) {
 	return s, nil
 }
 
+func storeNewClientView(db *db.DB, clientView json.RawMessage) error {
+	v, err := nomsjson.FromJSON(bytes.NewReader(clientView), db.Noms())
+	if err != nil {
+		return err
+	}
+	nm, ok := v.(types.Map)
+	if !ok {
+		return fmt.Errorf("clientview is not a Map, it's a %s", v.Kind().String())
+	}
+	// TODO fritz yes this is inefficient, will fix up Map so we don't have to go
+	// back and forth. But after it works.
+	m := kv.NewMapFromNoms(db.Noms(), nm)
+	if m == nil {
+		return errors.New("couldnt create a Map from a Noms Map")
+	}
+	return db.PutData(m.NomsMap(), types.String(m.Checksum().String()))
+}
+
 func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	verbose.SetVerbose(true)
 	log.Println("Handling request: ", r.URL.String())