Merge pull request #366 from 0xProject/feature/config-for-browsers

Update all configuration options in preparation for browser support

.circleci/config.yml

@@ -41,8 +41,11 @@ jobs:
           name: Run Go tests
           command: make test-go
       - run:
-          name: Run WebAssembly tests
-          command: make test-wasm
+          name: Run WebAssembly tests in Node.js
+          command: make test-wasm-node
+      - run:
+          name: Run WebAssembly tests in a headless browser
+          command: make test-wasm-browser
       - run:
           name: Test installing Mesh without CGO
           command: CGO_ENABLED=0 go install ./...

Gopkg.toml

@@ -101,15 +101,15 @@
   name = "github.com/ipfs/go-ds-leveldb"
   version = "0.0.2"
 
-[[constraint]]
-  branch = "support-arrays"
-  name = "github.com/norunners/vert"
-  source = "github.com/albrow/vert"
-
 [[constraint]]
   name = "github.com/libp2p/go-libp2p-core"
   version = "0.2.2"
 
 [[constraint]]
   name = "github.com/libp2p/go-libp2p-discovery"
   version = "0.1.0"
+
+[[override]]
+  name = "github.com/libp2p/go-ws-transport"
+  source = "github.com/0xProject/go-ws-transport"
+  revision = "163cee1e07594cd148a9086cd3cce5f901e4dae9"

Makefile

@@ -1,5 +1,5 @@
 .PHONY: deps
-deps: deps-go deps-js
+deps: deps-go deps-js wasmbrowsertest
 
 
 .PHONY: deps-go
@@ -12,9 +12,15 @@ deps-js:
 	yarn install
 
 
+# wasmbrowsertest is required for running WebAssembly tests in the browser.
+.PHONY: wasmbrowsertest
+wasmbrowsertest:
+	go get -u github.com/agnivade/wasmbrowsertest
+
+
 # Installs dependencies without updating Gopkg.lock or yarn.lock
 .PHONY: deps-no-lockfile
-deps-no-lockfile: deps-go-no-lockfile deps-js-no-lockfile
+deps-no-lockfile: deps-go-no-lockfile deps-js-no-lockfile wasmbrowsertest
 
 
 .PHONY: deps-go-no-lockfile
@@ -28,19 +34,24 @@ deps-js-no-lockfile:
 
 
 .PHONY: test-all
-test-all: test-go test-wasm
+test-all: test-go test-wasm-node test-wasm-browser
 
 
 .PHONY: test-go
 test-go:
 	go test ./... -race -timeout 30s
 
 
-.PHONY: test-wasm
-test-wasm:
+.PHONY: test-wasm-node
+test-wasm-node:
 	export ZEROEX_MESH_ROOT_DIR=$$(pwd); GOOS=js GOARCH=wasm go test -exec="$$ZEROEX_MESH_ROOT_DIR/test-wasm/go_js_wasm_exec" ./...
 
 
+.PHONY: test-wasm-browser
+test-wasm-browser:
+	GOOS=js GOARCH=wasm go test -tags=browser -exec="$$GOPATH/bin/wasmbrowsertest" ./...
+
+
 .PHONY: lint
 lint:
 	golangci-lint run
@@ -65,6 +76,7 @@ mesh-bootstrap:
 db-integrity-check:
 	go install ./cmd/db-integrity-check
 
+
 .PHONY: cut-release
 cut-release:
 	go run ./cmd/cut-release/main.go

cmd/mesh-bootstrap/Dockerfile

@@ -4,7 +4,7 @@
 #
 
 # mesh-builder produces a statically linked binary
-FROM golang:1.12.1-alpine3.9 as mesh-builder
+FROM golang:1.12.9-alpine3.9 as mesh-builder
 
 
 RUN apk update && apk add ca-certificates nodejs-current npm make git dep gcc build-base musl linux-headers

cmd/mesh-bootstrap/main.go

@@ -9,11 +9,10 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/libp2p/go-libp2p-core/peer"
-
 	"github.com/0xProject/0x-mesh/keys"
 	"github.com/0xProject/0x-mesh/loghooks"
 	"github.com/0xProject/0x-mesh/p2p"
@@ -24,6 +23,7 @@ import (
 	p2pcrypto "github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/host"
 	p2pnet "github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/routing"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
 	"github.com/libp2p/go-libp2p/p2p/host/relay"
@@ -57,9 +57,8 @@ type Config struct {
 	// P2PAdvertiseAddrs is a comma separated list of libp2p multiaddresses which the
 	// bootstrap node will advertise to peers.
 	P2PAdvertiseAddrs string `envvar:"P2P_ADVERTISE_ADDRS"`
-	// PrivateKey path is the path to a private key file which will be used for
-	// signing messages and generating a peer ID.
-	PrivateKeyPath string `envvar:"PRIVATE_KEY_PATH" default:"0x_mesh/keys/privkey"`
+	// DataDir is the directory used for storing data.
+	DataDir string `envvar:"DATA_DIR" default:"0x_mesh"`
 }
 
 func init() {
@@ -69,6 +68,18 @@ func init() {
 	relay.AdvertiseBootDelay = 30 * time.Second
 }
 
+func getPrivateKeyPath(config Config) string {
+	return filepath.Join(config.DataDir, "keys", "privkey")
+}
+
+func getDHTDir(config Config) string {
+	return filepath.Join(config.DataDir, "p2p", "dht")
+}
+
+func getPeerstoreDir(config Config) string {
+	return filepath.Join(config.DataDir, "p2p", "peerstore")
+}
+
 func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -86,7 +97,7 @@ func main() {
 	log.AddHook(loghooks.NewKeySuffixHook())
 
 	// Parse private key file and add peer ID log hook
-	privKey, err := initPrivateKey(config.PrivateKeyPath)
+	privKey, err := initPrivateKey(getPrivateKeyPath(config))
 	if err != nil {
 		log.WithField("error", err).Fatal("could not initialize private key")
 	}
@@ -101,7 +112,8 @@ func main() {
 	var kadDHT *dht.IpfsDHT
 	newDHT := func(h host.Host) (routing.PeerRouting, error) {
 		var err error
-		kadDHT, err = p2p.NewDHT(ctx, h)
+		dhtDir := getDHTDir(config)
+		kadDHT, err = p2p.NewDHT(ctx, dhtDir, h)
 		if err != nil {
 			log.WithField("error", err).Fatal("could not create DHT")
 		}

cmd/mesh/Dockerfile

@@ -4,7 +4,7 @@
 #
 
 # mesh-builder produces a statically linked binary
-FROM golang:1.12.1-alpine3.9 as mesh-builder
+FROM golang:1.12.9-alpine3.9 as mesh-builder
 
 
 RUN apk update && apk add ca-certificates nodejs-current npm make git dep gcc build-base musl linux-headers
@@ -29,8 +29,10 @@ COPY --from=mesh-builder /go/src/github.com/0xProject/0x-mesh/mesh /usr/mesh/mes
 ENV RPC_PORT=60557
 EXPOSE 60557
 
-ENV P2P_LISTEN_PORT=60558
+ENV P2P_TCP_PORT=60558
 EXPOSE 60558
+ENV P2P_WEBSOCKETS_PORT=60559
+EXPOSE 60559
 
 RUN chmod +x ./mesh
 

core/core.go

@@ -1,5 +1,3 @@
-// +build !js
-
 // Package core contains everything needed to configure and run a 0x Mesh node.
 package core
 
@@ -55,8 +53,12 @@ type Config struct {
 	// DataDir is the directory to use for persisting all data, including the
 	// database and private key files.
 	DataDir string `envvar:"DATA_DIR" default:"0x_mesh"`
-	// P2PListenPort is the port on which to listen for new peer connections.
-	P2PListenPort int `envvar:"P2P_LISTEN_PORT"`
+	// P2PTCPPort is the port on which to listen for new TCP connections from
+	// peers in the network. Set to 60558 by default.
+	P2PTCPPort int `envvar:"P2P_TCP_PORT" default:"60558"`
+	// P2PWebSocketsPort is the port on which to listen for new WebSockets
+	// connections from peers in the network. Set to 60559 by default.
+	P2PWebSocketsPort int `envvar:"P2P_WEBSOCKETS_PORT" default:"60559"`
 	// EthereumRPCURL is the URL of an Etheruem node which supports the JSON RPC
 	// API.
 	EthereumRPCURL string `envvar:"ETHEREUM_RPC_URL" json:"-"`
@@ -350,13 +352,14 @@ func (app *App) Start(ctx context.Context) error {
 	// Initialize the p2p node.
 	nodeConfig := p2p.Config{
 		Topic:            getPubSubTopic(app.config.EthereumNetworkID),
-		ListenPort:       app.config.P2PListenPort,
+		TCPPort:          app.config.P2PTCPPort,
+		WebSocketsPort:   app.config.P2PWebSocketsPort,
 		Insecure:         false,
 		PrivateKey:       app.privKey,
 		MessageHandler:   app,
 		RendezvousString: getRendezvous(app.config.EthereumNetworkID),
 		UseBootstrapList: app.config.UseBootstrapList,
-		PeerstoreDir:     filepath.Join(app.config.DataDir, "peerstore"),
+		DataDir:          filepath.Join(app.config.DataDir, "p2p"),
 	}
 	var err error
 	app.node, err = p2p.New(innerCtx, nodeConfig)

core/message_handler.go

@@ -1,5 +1,3 @@
-// +build !js
-
 package core
 
 import (

core/peer_score.go

@@ -1,5 +1,3 @@
-// +build !js
-
 package core
 
 import (

core/validation.go

@@ -1,5 +1,3 @@
-// +build !js
-
 package core
 
 import (

db/db.go

@@ -36,18 +36,6 @@ type DB struct {
 	colLock         sync.Mutex
 }
 
-// Open creates a new database using the given file path for permanent storage.
-// It is not safe to have multiple DBs using the same file path.
-func Open(path string) (*DB, error) {
-	ldb, err := leveldb.OpenFile(path, nil)
-	if err != nil {
-		return nil, err
-	}
-	return &DB{
-		ldb: ldb,
-	}, nil
-}
-
 // Close closes the database. It is not safe to call Close if there are any
 // other methods that have not yet returned. It is safe to call Close multiple
 // times.

db/open.go

@@ -0,0 +1,17 @@
+// +build !js
+
+package db
+
+import "github.com/syndtr/goleveldb/leveldb"
+
+// Open creates a new database using the given file path for permanent storage.
+// It is not safe to have multiple DBs using the same file path.
+func Open(path string) (*DB, error) {
+	ldb, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &DB{
+		ldb: ldb,
+	}, nil
+}

db/open_js.go

@@ -0,0 +1,20 @@
+// +build js,wasm
+
+package db
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+// Open creates a new database using in-memory storage. The given file path is
+// ignored.
+func Open(path string) (*DB, error) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return &DB{
+		ldb: ldb,
+	}, nil
+}

ethereum/blockwatch/block_watcher_test.go

@@ -1,3 +1,5 @@
+// +build !browser
+
 package blockwatch
 
 import (