fix: simplify data model, proofs_hashes table (#101)

api/migrations/migrations.go

@@ -116,4 +116,14 @@ var Migrations = []migrate.Migration{
 		CREATE INDEX proofs_arr_idx ON trees_proofs USING GIN ((proofs_array(proofs)));
 		`,
 	},
+	{
+		Name: "2023-06-26.0.proofs_hashes.sql",
+		SQL: `
+		CREATE TABLE IF NOT EXISTS proofs_hashes (
+			hash bytea,
+			root bytea
+		);
+		CREATE INDEX IF NOT EXISTS proofs_hashes_hash_idx ON proofs_hashes (hash);
+		`,
+	},
 }

api/migrations/scripts/000-rebuild-proofs-hashes/main.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sync"
+
+	"github.com/contextwtf/lanyard/merkle"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
+	"golang.org/x/sync/errgroup"
+)
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "processor error: %s", err)
+		debug.PrintStack()
+		os.Exit(1)
+	}
+}
+
+func hashProof(p [][]byte) []byte {
+	return crypto.Keccak256(p...)
+}
+
+func migrateTree(
+	ctx context.Context,
+	tx pgx.Tx,
+	leaves [][]byte,
+) error {
+	tree := merkle.New(leaves)
+
+	var (
+		proofHashes = [][]any{}
+		eg          errgroup.Group
+		pm          sync.Mutex
+	)
+	eg.SetLimit(runtime.NumCPU())
+
+	for _, l := range leaves {
+		l := l //avoid capture
+		eg.Go(func() error {
+			pf := tree.Proof(l)
+			if !merkle.Valid(tree.Root(), pf, l) {
+				return errors.New("invalid proof for tree")
+			}
+			proofHash := hashProof(pf)
+			pm.Lock()
+			proofHashes = append(proofHashes, []any{tree.Root(), proofHash})
+			pm.Unlock()
+			return nil
+		})
+	}
+	err := eg.Wait()
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.CopyFrom(ctx, pgx.Identifier{"proofs_hashes"},
+		[]string{"root", "hash"},
+		pgx.CopyFromRows(proofHashes),
+	)
+
+	return err
+}
+
+func main() {
+	ctx := context.Background()
+	const defaultPGURL = "postgres:///al"
+	dburl := os.Getenv("DATABASE_URL")
+	if dburl == "" {
+		dburl = defaultPGURL
+	}
+	dbc, err := pgxpool.ParseConfig(dburl)
+	check(err)
+
+	db, err := pgxpool.ConnectConfig(ctx, dbc)
+	check(err)
+
+	log.Println("fetching roots from db")
+	const q = `
+		SELECT unhashed_leaves
+		FROM trees
+		WHERE root not in (select root from proofs_hashes group by 1)
+	`
+	rows, err := db.Query(ctx, q)
+	check(err)
+	defer rows.Close()
+
+	trees := [][][]byte{}
+
+	for rows.Next() {
+		var t [][]byte
+		err := rows.Scan(&t)
+		trees = append(trees, t)
+		check(err)
+	}
+
+	if len(trees) == 0 {
+		log.Println("no trees to process")
+		return
+	}
+
+	log.Printf("migrating %d trees", len(trees))
+
+	tx, err := db.Begin(ctx)
+	check(err)
+	defer tx.Rollback(ctx)
+
+	var count int
+
+	for _, tree := range trees {
+		err = migrateTree(ctx, tx, tree)
+		check(err)
+		count++
+		if count%1000 == 0 {
+			log.Printf("migrated %d/%d trees", count, len(trees))
+		}
+	}
+
+	log.Printf("committing %d trees", len(trees))
+	err = tx.Commit(ctx)
+	check(err)
+	log.Printf("done")
+}

api/proof.go

@@ -1,9 +1,11 @@
 package api
 
 import (
+	"bytes"
 	"errors"
 	"net/http"
 
+	"github.com/contextwtf/lanyard/merkle"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/jackc/pgx/v4"
@@ -18,41 +20,19 @@ func (s *Server) GetProof(w http.ResponseWriter, r *http.Request) {
 	var (
 		ctx  = r.Context()
 		root = common.FromHex(r.URL.Query().Get("root"))
-		leaf = r.URL.Query().Get("unhashedLeaf")
-		addr = r.URL.Query().Get("address")
+		leaf = common.FromHex(r.URL.Query().Get("unhashedLeaf"))
+		addr = common.HexToAddress(r.URL.Query().Get("address"))
 	)
 	if len(root) == 0 {
 		s.sendJSONError(r, w, nil, http.StatusBadRequest, "missing root")
 		return
 	}
-	if leaf == "" && addr == "" {
+	if len(leaf) == 0 && addr == (common.Address{}) {
 		s.sendJSONError(r, w, nil, http.StatusBadRequest, "missing leaf")
 		return
 	}
 
-	const q = `
-		WITH tree AS (
-			SELECT jsonb_array_elements(proofs) proofs
-			FROM trees
-			WHERE root = $1
-		)
-		SELECT
-			proofs->'leaf',
-			proofs->'proof'
-		FROM tree
-		WHERE (
-			--eth addresses contain mixed casing to
-			--accommodate checksums. we sidestep
-			--the casing issues for user queries
-			lower(proofs->>'addr') = lower($2)
-			OR lower(proofs->>'leaf') = lower($3)
-		)
-	`
-	var (
-		resp = &getProofResp{}
-		row  = s.db.QueryRow(ctx, q, root, addr, leaf)
-		err  = row.Scan(&resp.UnhashedLeaf, &resp.Proof)
-	)
+	td, err := getTree(ctx, s.db, root)
 	if errors.Is(err, pgx.ErrNoRows) {
 		s.sendJSONError(r, w, nil, http.StatusNotFound, "tree not found")
 		w.Header().Set("Cache-Control", "public, max-age=60")
@@ -62,12 +42,49 @@ func (s *Server) GetProof(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	var (
+		leaves [][]byte
+		target []byte
+	)
+	// check if leaf is in tree and error if not
+	for _, l := range td.UnhashedLeaves {
+		if len(target) == 0 {
+			if len(leaf) > 0 {
+				if bytes.Equal(l, leaf) {
+					target = l
+				}
+			} else if leaf2Addr(l, td.Ltd, td.Packed).Hex() == addr.Hex() {
+				target = l
+			}
+		}
+
+		leaves = append(leaves, l)
+	}
+
+	if len(target) == 0 {
+		s.sendJSONError(r, w, nil, http.StatusNotFound, "leaf not found in tree")
+		return
+	}
+
+	var (
+		p    = merkle.New(leaves).Proof(target)
+		phex = []hexutil.Bytes{}
+	)
+
+	// convert [][]byte to []hexutil.Bytes
+	for _, p := range p {
+		phex = append(phex, p)
+	}
+
 	// cache for 1 year if we're returning an unhashed leaf proof
 	// or 60 seconds for an address proof
-	if leaf != "" {
+	if len(leaf) > 0 {
 		w.Header().Set("Cache-Control", "public, max-age=31536000")
 	} else {
 		w.Header().Set("Cache-Control", "public, max-age=60")
 	}
-	s.sendJSON(r, w, resp)
+	s.sendJSON(r, w, getProofResp{
+		UnhashedLeaf: target,
+		Proof:        phex,
+	})
 }

api/root.go

@@ -1,31 +1,13 @@
 package api
 
 import (
-	"encoding/json"
 	"net/http"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/jackc/pgx/v4"
 )
 
-func proofURLToDBQuery(param string) string {
-	type proofLookup struct {
-		Proof []string `json:"proof"`
-	}
-
-	lookup := proofLookup{
-		Proof: strings.Split(param, ","),
-	}
-
-	q, err := json.Marshal([]proofLookup{lookup})
-	if err != nil {
-		return ""
-	}
-
-	return string(q)
-}
-
 func (s *Server) GetRoot(w http.ResponseWriter, r *http.Request) {
 	type rootResp struct {
 		Root hexutil.Bytes `json:"root"`
@@ -37,24 +19,40 @@ func (s *Server) GetRoot(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var (
-		ctx     = r.Context()
-		proof   = r.URL.Query().Get("proof")
-		dbQuery = proofURLToDBQuery(proof)
+		ctx   = r.Context()
+		err   error
+		proof = r.URL.Query().Get("proof")
+		ps    = strings.Split(proof, ",")
+		pb    = [][]byte{}
 	)
-	if proof == "" || dbQuery == "" {
-		s.sendJSONError(r, w, nil, http.StatusBadRequest, "missing list of proofs")
+
+	for _, s := range ps {
+		var b []byte
+		b, err = hexutil.Decode(s)
+		if err != nil {
+			break
+		}
+		pb = append(pb, b)
+	}
+
+	if len(pb) == 0 || err != nil {
+		s.sendJSONError(r, w, nil, http.StatusBadRequest, "missing or malformed list of proofs")
 		return
 	}
 
 	const q = `
 		SELECT root
-		FROM trees_proofs
-		WHERE proofs_array(proofs) @> proofs_array($1);
+		FROM proofs_hashes
+		WHERE hash = $1
+		group by 1;
 	`
-	roots := make([]hexutil.Bytes, 0)
-	rb := make(hexutil.Bytes, 0)
+	var (
+		roots []hexutil.Bytes
+		rb    hexutil.Bytes
+		ph    = hashProof(pb)
+	)
 
-	_, err := s.db.QueryFunc(ctx, q, []interface{}{dbQuery}, []interface{}{&rb}, func(qfr pgx.QueryFuncRow) error {
+	_, err = s.db.QueryFunc(ctx, q, []interface{}{&ph}, []interface{}{&rb}, func(qfr pgx.QueryFuncRow) error {
 		roots = append(roots, rb)
 		return nil
 	})