Skip to content
Permalink
Browse files

zkLedger as run in NVV18 (zkledger.pdf)

  • Loading branch information...
narula committed Feb 14, 2019
1 parent e139378 commit f0583d13c92eef2fc10845b2966f758b006d8dc5
Showing with 6,047 additions and 0 deletions.
  1. +13 −0 .gitignore
  2. +31 −0 DEVELOPERS.md
  3. +20 −0 README.md
  4. +299 −0 audit.go
  5. +193 −0 audit_test.go
  6. +775 −0 bank.go
  7. +405 −0 bank_test.go
  8. +144 −0 clients.go
  9. +216 −0 cmd/apl-auditor/auditor.go
  10. +224 −0 cmd/apl-bank/bank.go
  11. +55 −0 cmd/apl-ledger/ledger.go
  12. +24 −0 cmd/keygen/keygen.go
  13. +12 −0 cmd/setup/.gitignore
  14. +658 −0 cmd/setup/apl_env.go
  15. +1 −0 cmd/setup/keys/b0_pk
  16. +1 −0 cmd/setup/keys/b0_sk
  17. +1 −0 cmd/setup/keys/b10_pk
  18. BIN cmd/setup/keys/b10_sk
  19. +1 −0 cmd/setup/keys/b11_pk
  20. +1 −0 cmd/setup/keys/b11_sk
  21. +1 −0 cmd/setup/keys/b12_pk
  22. +1 −0 cmd/setup/keys/b12_sk
  23. +1 −0 cmd/setup/keys/b13_pk
  24. +1 −0 cmd/setup/keys/b13_sk
  25. +1 −0 cmd/setup/keys/b14_pk
  26. +1 −0 cmd/setup/keys/b14_sk
  27. +1 −0 cmd/setup/keys/b15_pk
  28. +1 −0 cmd/setup/keys/b15_sk
  29. +1 −0 cmd/setup/keys/b16_pk
  30. +1 −0 cmd/setup/keys/b16_sk
  31. +1 −0 cmd/setup/keys/b17_pk
  32. +1 −0 cmd/setup/keys/b17_sk
  33. +1 −0 cmd/setup/keys/b18_pk
  34. +1 −0 cmd/setup/keys/b18_sk
  35. +1 −0 cmd/setup/keys/b19_pk
  36. +1 −0 cmd/setup/keys/b19_sk
  37. +1 −0 cmd/setup/keys/b1_pk
  38. +1 −0 cmd/setup/keys/b1_sk
  39. +1 −0 cmd/setup/keys/b20_pk
  40. +1 −0 cmd/setup/keys/b20_sk
  41. +1 −0 cmd/setup/keys/b21_pk
  42. BIN cmd/setup/keys/b21_sk
  43. +1 −0 cmd/setup/keys/b22_pk
  44. +1 −0 cmd/setup/keys/b22_sk
  45. +1 −0 cmd/setup/keys/b23_pk
  46. +1 −0 cmd/setup/keys/b23_sk
  47. +1 −0 cmd/setup/keys/b24_pk
  48. BIN cmd/setup/keys/b24_sk
  49. +1 −0 cmd/setup/keys/b25_pk
  50. +1 −0 cmd/setup/keys/b25_sk
  51. +1 −0 cmd/setup/keys/b26_pk
  52. +1 −0 cmd/setup/keys/b26_sk
  53. +1 −0 cmd/setup/keys/b27_pk
  54. +1 −0 cmd/setup/keys/b27_sk
  55. +1 −0 cmd/setup/keys/b28_pk
  56. +1 −0 cmd/setup/keys/b28_sk
  57. +1 −0 cmd/setup/keys/b29_pk
  58. +1 −0 cmd/setup/keys/b29_sk
  59. +1 −0 cmd/setup/keys/b2_pk
  60. +1 −0 cmd/setup/keys/b2_sk
  61. +1 −0 cmd/setup/keys/b30_pk
  62. +1 −0 cmd/setup/keys/b30_sk
  63. +1 −0 cmd/setup/keys/b31_pk
  64. +1 −0 cmd/setup/keys/b31_sk
  65. +1 −0 cmd/setup/keys/b32_pk
  66. BIN cmd/setup/keys/b32_sk
  67. +1 −0 cmd/setup/keys/b33_pk
  68. BIN cmd/setup/keys/b33_sk
  69. +1 −0 cmd/setup/keys/b34_pk
  70. +1 −0 cmd/setup/keys/b34_sk
  71. +1 −0 cmd/setup/keys/b35_pk
  72. +1 −0 cmd/setup/keys/b35_sk
  73. +1 −0 cmd/setup/keys/b36_pk
  74. +1 −0 cmd/setup/keys/b36_sk
  75. +1 −0 cmd/setup/keys/b37_pk
  76. +1 −0 cmd/setup/keys/b37_sk
  77. +1 −0 cmd/setup/keys/b38_pk
  78. +1 −0 cmd/setup/keys/b38_sk
  79. +1 −0 cmd/setup/keys/b39_pk
  80. +1 −0 cmd/setup/keys/b39_sk
  81. +1 −0 cmd/setup/keys/b3_pk
  82. +1 −0 cmd/setup/keys/b3_sk
  83. +1 −0 cmd/setup/keys/b40_pk
  84. +2 −0 cmd/setup/keys/b40_sk
  85. +1 −0 cmd/setup/keys/b41_pk
  86. +1 −0 cmd/setup/keys/b41_sk
  87. +1 −0 cmd/setup/keys/b42_pk
  88. +1 −0 cmd/setup/keys/b42_sk
  89. +1 −0 cmd/setup/keys/b43_pk
  90. +1 −0 cmd/setup/keys/b43_sk
  91. +1 −0 cmd/setup/keys/b44_pk
  92. +1 −0 cmd/setup/keys/b44_sk
  93. +1 −0 cmd/setup/keys/b45_pk
  94. +1 −0 cmd/setup/keys/b45_sk
  95. +1 −0 cmd/setup/keys/b46_pk
  96. +1 −0 cmd/setup/keys/b46_sk
  97. +1 −0 cmd/setup/keys/b47_pk
  98. BIN cmd/setup/keys/b47_sk
  99. +1 −0 cmd/setup/keys/b48_pk
  100. +1 −0 cmd/setup/keys/b48_sk
  101. +1 −0 cmd/setup/keys/b49_pk
  102. +1 −0 cmd/setup/keys/b49_sk
  103. +1 −0 cmd/setup/keys/b4_pk
  104. +1 −0 cmd/setup/keys/b4_sk
  105. +1 −0 cmd/setup/keys/b5_pk
  106. +1 −0 cmd/setup/keys/b5_sk
  107. +1 −0 cmd/setup/keys/b6_pk
  108. +1 −0 cmd/setup/keys/b6_sk
  109. +1 −0 cmd/setup/keys/b7_pk
  110. BIN cmd/setup/keys/b7_sk
  111. +1 −0 cmd/setup/keys/b8_pk
  112. +1 −0 cmd/setup/keys/b8_sk
  113. +1 −0 cmd/setup/keys/b9_pk
  114. BIN cmd/setup/keys/b9_sk
  115. +1 −0 cmd/setup/keys/is_pk
  116. +1 −0 cmd/setup/keys/is_sk
  117. +216 −0 cmd/setup/main.go
  118. +1,229 −0 crypto.go
  119. +716 −0 crypto_test.go
  120. +14 −0 debug.go
  121. +225 −0 ledger.go
  122. +38 −0 ledger_test.go
  123. +175 −0 pki.go
  124. +64 −0 testutil.go
  125. +206 −0 transaction.go
@@ -0,0 +1,13 @@
*~
zkledger.test
cmd/apl-bank/apl-bank
cmd/apl-bank/keys
cmd/apl-ledger/apl-ledger
cmd/apl-auditor/apl-auditor
cmd/apl-auditor/keys
cmd/keygen/keygen
cmd/keygen/keys
*.byte
*.exe
*.out
*.prof
@@ -0,0 +1,31 @@
# zkLedger File Structure

### Top Level

All the zkLedger source code is contained in the top level directory. This includes auditor, bank, and ledger functionality.

- **audit.go**
- **bank.go**
- **clients.go**
- **debug.go**
- **ledger.go**
- **pki.go**
- **system_test.go**
- **testutil.go**
- **transaction.go**

### cmd/

This directory contains the control infrastructure to actually run a zkLedger instance.

- **apl-auditor/**
- **apl-bank/**
- **apl-ledger/**
- **keygen/**
- **setup/**

To run experiments, `setup/` contains a testing harness for local and remote tests.

- `main.go`: contains preset tests (e.g. r50TX_herf where every bank you pass in performs 50 transactions and at the end the Herfindahl index is calculated between them) but also has the flexibility to insert others. It also takes in parameters to for a remote testing setup.
- `apl_env.go`: contains methods to create the environment for zkLedger to execute, either locally or remotely.
- `keys/`: **DO NOT USE THESE IN PRODUCTION ENVIRONMENTS** contains a collection of 50 bank public/private key pairs for testing.
@@ -0,0 +1,20 @@
# zkLedger

zkLedger is a design for a ledger which has private transactions, but supports provably-correct queries over the ledger.

Maybe you'd like to run the tests:

```
cd $GOPATH/src/github.com/mit-dci/apl
go test
```

Or, run a local experiment with a few banks, a single-server ledger, and an auditor:


```
cd cmd/setup
./setup -t simple1
```

(You can add `-debug` to get a lot of timing information and other debug output)
299 audit.go
@@ -0,0 +1,299 @@
package zkledger

import (
"fmt"
"log"
"math/big"
"net"
"net/rpc"
"sync"
)

type Auditor struct {
num int
local_ledger *LocalLedger
receivedTxns chan *EncryptedTransaction
mu *sync.Mutex
lastSeen int
Done chan bool
banks []BankClient
pki *PKI
CommsCache []ECPoint
RTokenCache []ECPoint
Setup chan struct{}
}

func MakeAuditor(num int, pki *PKI) *Auditor {
a := &Auditor{
num: num,
local_ledger: MakeLocalLedger(),
receivedTxns: make(chan *EncryptedTransaction, TXN_BUFFER),
mu: new(sync.Mutex),
lastSeen: -1,
Done: make(chan bool),
pki: pki,
Setup: make(chan struct{}),
}
a.CommsCache = make([]ECPoint, a.num)
a.RTokenCache = make([]ECPoint, a.num)
for i := 0; i < a.num; i++ {
a.CommsCache[i] = EC.Zero()
a.RTokenCache[i] = EC.Zero()
}
go a.start()
return a
}

func (a *Auditor) Go(c APLClientConfig, _ *struct{}) error {
go a.register(c.Hostname, c.BasePort, c.BankHostnames)
a.listen(c.Hostname, c.BasePort)
return nil
}

func (a *Auditor) listen(hostname string, basePort int) {
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", basePort+a.num+2))
if err != nil {
log.Fatalf("[A] Could not listen %v\n", err)
}
err = rpc.Register(a)
if err != nil {
panic(err)
}
for {
conn, err := listener.Accept()
if err != nil {
fmt.Println(err)
continue
}
go rpc.ServeConn(conn)
}
}

func (a *Auditor) register(hostname string, baseport int, bankHostnames []string) {
var wg sync.WaitGroup
a.banks = make([]BankClient, a.num)
for i := 0; i < a.num; i++ {
wg.Add(1)
go func(i int) {
x := MakeRemoteBankClient()
x.connect(bankHostnames[i], baseport+i+1)
a.banks[i] = x
wg.Done()
}(i)
}
wg.Wait()
Dprintf("[A] Registered with banks\n")
close(a.Setup)
}

func (a *Auditor) start() {
<-a.Setup
Dprintf("[A] Starting audit loop...\n")
var pks []ECPoint
for {
select {
case etx := <-a.receivedTxns:
Dprintf("[A][%v] Received txn...\n", etx.Index)
// Verify
if pks == nil {
pks = make([]ECPoint, a.num+1) // all the banks and the issuer
for i := 0; i < a.num+1; i++ {
pks[i] = a.pki.Get(i)
}
}
if *emptyTxn {
continue
}
a.mu.Lock()
if !etx.Verify(pks, a.CommsCache, a.RTokenCache, "A") {
log.Fatalf("[A][%v] Bad transaction!\n", etx.Index)
}
Dprintf("[A][%v] Verified txn...\n", etx.Index)
lastSeen := a.lastSeen
a.mu.Unlock()
if etx.Index < lastSeen {
log.Fatalf("[A] lastSeen %v out of whack with received transactions %v\n", lastSeen, etx.Index)
} else if etx.Index == lastSeen {
log.Fatalf("[A] lastSeen %v out of whack with received transactions %v\n", lastSeen, etx.Index)
} else if etx.Index == lastSeen+1 {
a.mu.Lock()
a.lastSeen = a.lastSeen + 1
etx.reduce()
a.local_ledger.add(etx)
if etx.Type == Transfer {
for i := 0; i < len(etx.Entries); i++ {
//Dprintf("[A] Adding RToken %v...\n", etx.Entries[i].RToken)
a.RTokenCache[i] = a.RTokenCache[i].Add(etx.Entries[i].RToken)
a.CommsCache[i] = a.CommsCache[i].Add(etx.Entries[i].Comm)
}
} else if etx.Type == Issuance || etx.Type == Withdrawal {
// Only one bank for now
en := &etx.Entries[etx.Sender]
gval := EC.G.Mult(en.V)
a.CommsCache[etx.Sender] = EC.Add(a.CommsCache[etx.Sender], gval)
}
Dprintf("[A][%v] Processed txn\n", etx.Index)
a.mu.Unlock()
} else {
Dprintf("[A][%v] Received txn out of order, expected %v\n", etx.Index, a.lastSeen)
a.receivedTxns <- etx
}
case <-a.Done:
Dprintf("[A] Shutting down audit loop...\n")
close(a.receivedTxns)
return
}
}
}

func (a *Auditor) Stop(_ *struct{}, _ *struct{}) error {
a.Done <- true
return nil
}

func (a *Auditor) Notify(etx *EncryptedTransaction, _ *struct{}) error {
Dprintf("[A][%v] Notified of txn\n", etx.Index)
a.receivedTxns <- etx
return nil
}

// Compute # of asset for a given bank according to the ledger.
func (a *Auditor) computeSum(bank_i int) (*big.Int, bool) {
Dprintf("[A] Auditing bank %v \n", bank_i)
var rep AuditRep
a.banks[bank_i].Audit(&struct{}{}, &rep)
comms := EC.Zero()
rtokens := EC.Zero()
a.mu.Lock()
if *useCache {
comms = a.CommsCache[bank_i]
rtokens = a.RTokenCache[bank_i]
} else {
for i := 0; i < len(a.local_ledger.Transactions); i++ {
etx := &a.local_ledger.Transactions[i]
if etx.Type == Transfer {
comms = EC.Add(comms, etx.Entries[bank_i].Comm)
rtokens = EC.Add(rtokens, etx.Entries[bank_i].RToken)
} else if (etx.Type == Issuance || etx.Type == Withdrawal) && etx.Sender == bank_i {
gval := EC.G.Mult(etx.Entries[etx.Sender].V)
comms = EC.Add(comms, gval)
}
}
}
a.mu.Unlock()
gv := EC.G.Mult(rep.Sum).Neg() // 1 / g^\sum{v_i}
T := EC.Add(comms, gv)
verifies := VerifyEquivalence(T, rtokens, EC.H, a.pki.Get(bank_i), rep.Eproof)
if !verifies {
Dprintf("[A] Bank %v proof didn't verify! Their total: %v\n", bank_i, rep.Sum)
Dprintf(" My \\sum{rtks_i}: %v\n", rtokens)
Dprintf(" My \\sum{comms_i}: %v\n", comms)
Dprintf(" gv: %v\n", gv)
Dprintf(" T: %v\n", T)
}
return rep.Sum, verifies
}

// Should hold a.mu. OK to call in parallel for different banks.
func (a *Auditor) sumOneBank(wg *sync.WaitGroup, bank_i int, totals []*big.Int, cache bool) {
var rep AuditRep
a.banks[bank_i].Audit(&struct{}{}, &rep)
comms := EC.Zero()
rtokens := EC.Zero()
if *useCache && cache {
comms = a.CommsCache[bank_i]
rtokens = a.RTokenCache[bank_i]
} else {
for i := 0; i < len(a.local_ledger.Transactions); i++ {
etx := &a.local_ledger.Transactions[i]
if etx.Type == Transfer {
comms = EC.Add(comms, etx.Entries[bank_i].Comm)
rtokens = EC.Add(rtokens, etx.Entries[bank_i].RToken)
} else if (etx.Type == Issuance || etx.Type == Withdrawal) && etx.Sender == bank_i {
gval := EC.G.Mult(etx.Entries[etx.Sender].V)
comms = EC.Add(comms, gval)
}
}
}
gv := EC.G.Mult(rep.Sum).Neg() // 1 / g^\sum{v_i}
T := EC.Add(comms, gv)
verifies := VerifyEquivalence(T, rtokens, EC.H, a.pki.Get(bank_i), rep.Eproof)
if !verifies {
Dprintf("[A] Bank %v proof didn't verify! Their total: %v\n", bank_i, rep.Sum)
Dprintf(" My \\sum{rtks_i}: %v\n", rtokens)
Dprintf(" My \\sum{comms_i}: %v\n", comms)
Dprintf(" gv: %v\n", gv)
Dprintf(" T: %v\n", T)
}
totals[bank_i] = rep.Sum
wg.Done()
}

func (a *Auditor) Herfindahl(cache bool, _ *struct{}) (*big.Rat, error) {
<-a.Setup
totals := make([]*big.Int, a.num)
total := big.NewInt(0)
concentrations := make([]*big.Rat, a.num)
var wg sync.WaitGroup
a.mu.Lock()
defer a.mu.Unlock()
wg.Add(a.num)
for i := 0; i < a.num; i++ {
go a.sumOneBank(&wg, i, totals, cache)
concentrations[i] = new(big.Rat)
}
wg.Wait()
for i := 0; i < a.num; i++ {
total.Add(total, totals[i])
}
Dprintf("[A] Herfindahl: %v totals, %v total\n", totals, total)
hIndex := big.NewRat(0, 1)
for i := 0; i < a.num; i++ {
marketShare := new(big.Rat).Quo(new(big.Rat).SetInt(totals[i]), new(big.Rat).SetInt(total))

hIndex.Add(hIndex, new(big.Rat).Mul(marketShare, marketShare)) // add the sum of squares of the market share
}
return hIndex, nil
}

// Compute # of asset for a given bank according to unencrypted test
// values in the ledger. ONLY TO BE USED FOR TESTING.
func (a *Auditor) computeClearSum(bank_i int) *big.Int {
total := big.NewInt(0)
for i := 0; i < len(a.local_ledger.Transactions); i++ {
total.Add(total, a.local_ledger.Transactions[i].Entries[bank_i].V)
}
return total
}

// Compute # total outstanding of an asset.
func (a *Auditor) computeOutstanding() *big.Int {
total := big.NewInt(0)
for i := 0; i < a.num; i++ {
v, err := a.computeSum(i)
if !err {
log.Fatalf("Commitments and rvals did not match for bank %v\n", i)
}
total.Add(total, v)
}
return total
}

func (a *Auditor) Audit(_ *struct{}, _ *struct{}) *big.Int {
<-a.Setup
Dprintf("[A] Auditing all banks \n")
return a.computeOutstanding()
}

func (a *Auditor) GetNumTX(_ *struct{}, _ *struct{}) int {
return len(a.local_ledger.Transactions)
}

// X Amount of some risky thing outstanding (outstanding sum)
// X Amount of some risky thing at one bank (sum)
// - Aggregate risk exposures (outstanding sum?)
// X Herfindahl concentration index
// - Aggregate leverage
// - Margin-to-equity ratios
// - Leverage ratios
// - Average correlations between *changes* in securities holdings
Oops, something went wrong.

0 comments on commit f0583d1

Please sign in to comment.
You can’t perform that action at this time.