implement access control lists for gateway

blocks/key/key_set.go

@@ -6,41 +6,73 @@ import (
 
 type KeySet interface {
 	Add(Key)
+	Has(Key) bool
 	Remove(Key)
 	Keys() []Key
 }
 
-type ks struct {
-	lock sync.RWMutex
-	data map[Key]struct{}
+type keySet struct {
+	keys map[Key]struct{}
 }
 
-func NewKeySet() KeySet {
-	return &ks{
-		data: make(map[Key]struct{}),
+func NewKeySet() *keySet {
+	return &keySet{make(map[Key]struct{})}
+}
+
+func (gcs *keySet) Add(k Key) {
+	gcs.keys[k] = struct{}{}
+}
+
+func (gcs *keySet) Has(k Key) bool {
+	_, has := gcs.keys[k]
+	return has
+}
+
+func (ks *keySet) Keys() []Key {
+	var out []Key
+	for k, _ := range ks.keys {
+		out = append(out, k)
 	}
+	return out
+}
+
+func (ks *keySet) Remove(k Key) {
+	delete(ks.keys, k)
 }
 
-func (wl *ks) Add(k Key) {
-	wl.lock.Lock()
-	defer wl.lock.Unlock()
+// TODO: implement disk-backed keyset for working with massive DAGs
 
-	wl.data[k] = struct{}{}
+type threadsafe struct {
+	lk sync.Mutex
+	ks KeySet
 }
 
-func (wl *ks) Remove(k Key) {
-	wl.lock.Lock()
-	defer wl.lock.Unlock()
+func Threadsafe(ks KeySet) *threadsafe {
+	return &threadsafe{ks: ks}
+}
 
-	delete(wl.data, k)
+func (ts *threadsafe) Has(k Key) bool {
+	ts.lk.Lock()
+	out := ts.ks.Has(k)
+	ts.lk.Unlock() //defer is slow
+	return out
 }
 
-func (wl *ks) Keys() []Key {
-	wl.lock.RLock()
-	defer wl.lock.RUnlock()
-	keys := make([]Key, 0)
-	for k := range wl.data {
-		keys = append(keys, k)
-	}
+func (ts *threadsafe) Remove(k Key) {
+	ts.lk.Lock()
+	ts.ks.Remove(k)
+	ts.lk.Unlock() //defer is slow
+}
+
+func (ts *threadsafe) Add(k Key) {
+	ts.lk.Lock()
+	ts.ks.Add(k)
+	ts.lk.Unlock() //defer is slow
+}
+
+func (ts *threadsafe) Keys() []Key {
+	ts.lk.Lock()
+	keys := ts.ks.Keys()
+	ts.lk.Unlock() //defer is slow
 	return keys
 }

cmd/ipfs/daemon.go

@@ -1,20 +1,21 @@
 package main
 
 import (
+	"encoding/json"
 	_ "expvar"
 	"fmt"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"sort"
-	"strings"
 	"sync"
 
 	_ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/codahale/metrics/runtime"
 	ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net"
 
+	key "github.com/ipfs/go-ipfs/blocks/key"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
 	commands "github.com/ipfs/go-ipfs/core/commands"
@@ -310,23 +311,20 @@ func serveHTTPApi(req cmds.Request) (error, <-chan error) {
 		return fmt.Errorf("serveHTTPApi: Option(%s) failed: %s", unrestrictedApiAccessKwd, err), nil
 	}
 
+	var allowlist key.KeySet
+	if !unrestricted {
+		allowlist = key.Threadsafe(key.NewKeySet())
+		for _, webuipath := range corehttp.WebUIPaths {
+			// extract the key
+			allowlist.Add(key.B58KeyDecode(webuipath[6:]))
+		}
+	}
+
 	apiGw := corehttp.NewGateway(corehttp.GatewayConfig{
-		Writable: true,
-		BlockList: &corehttp.BlockList{
-			Decider: func(s string) bool {
-				if unrestricted {
-					return true
-				}
-				// for now, only allow paths in the WebUI path
-				for _, webuipath := range corehttp.WebUIPaths {
-					if strings.HasPrefix(s, webuipath) {
-						return true
-					}
-				}
-				return false
-			},
-		},
+		Writable:  true,
+		AllowList: allowlist,
 	})
+
 	var opts = []corehttp.ServeOption{
 		corehttp.CommandsOption(*req.InvocContext()),
 		corehttp.WebUIOption,
@@ -401,11 +399,36 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) {
 		fmt.Printf("Gateway (readonly) server listening on %s\n", gatewayMaddr)
 	}
 
+	var denylist key.KeySet
+	var allowlist key.KeySet
+	if len(cfg.Gateway.DenyList) > 0 {
+		l, err := loadKeySetFromURLs(cfg.Gateway.DenyList)
+		if err != nil {
+			return err, nil
+		}
+		denylist = l
+	}
+
+	if len(cfg.Gateway.AllowList) > 0 {
+		l, err := loadKeySetFromURLs(cfg.Gateway.AllowList)
+		if err != nil {
+			return err, nil
+		}
+		allowlist = l
+	}
+
+	gateway := corehttp.Gateway{
+		Config: corehttp.GatewayConfig{
+			DenyList:  denylist,
+			AllowList: allowlist,
+		},
+	}
+
 	var opts = []corehttp.ServeOption{
 		corehttp.CommandsROOption(*req.InvocContext()),
 		corehttp.VersionOption(),
 		corehttp.IPNSHostnameOption(),
-		corehttp.GatewayOption(writable),
+		gateway.ServeOption(),
 	}
 
 	if len(cfg.Gateway.RootRedirect) > 0 {
@@ -425,6 +448,35 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) {
 	return nil, errc
 }
 
+type listing struct {
+	Uri  string
+	Keys []string
+}
+
+func loadKeySetFromURLs(urls []string) (key.KeySet, error) {
+	ks := key.NewKeySet()
+	for _, url := range urls {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+
+		var list listing
+		err = json.NewDecoder(resp.Body).Decode(&list)
+		if err != nil {
+			return nil, err
+		}
+		for _, k := range list.Keys {
+			lk := key.B58KeyDecode(k)
+			if lk == "" {
+				return nil, fmt.Errorf("incorrectly formatted key '%s'", k)
+			}
+			ks.Add(lk)
+		}
+	}
+	return key.Threadsafe(ks), nil
+}
+
 //collects options and opens the fuse mountpoint
 func mountFuse(req cmds.Request) error {
 	cfg, err := req.InvocContext().GetConfig()

core/corehttp/gateway.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"sync"
 
+	key "github.com/ipfs/go-ipfs/blocks/key"
 	core "github.com/ipfs/go-ipfs/core"
 	id "github.com/ipfs/go-ipfs/p2p/protocol/identify"
 )
@@ -17,8 +17,10 @@ type Gateway struct {
 
 type GatewayConfig struct {
 	Headers   map[string][]string
-	BlockList *BlockList
-	Writable  bool
+	DenyList  key.KeySet
+	AllowList key.KeySet
+
+	Writable bool
 }
 
 func NewGateway(conf GatewayConfig) *Gateway {
@@ -44,8 +46,7 @@ func (g *Gateway) ServeOption() ServeOption {
 
 func GatewayOption(writable bool) ServeOption {
 	g := NewGateway(GatewayConfig{
-		Writable:  writable,
-		BlockList: &BlockList{},
+		Writable: writable,
 	})
 	return g.ServeOption()
 }
@@ -59,33 +60,3 @@ func VersionOption() ServeOption {
 		return mux, nil
 	}
 }
-
-// Decider decides whether to Allow string
-type Decider func(string) bool
-
-type BlockList struct {
-	mu      sync.RWMutex
-	Decider Decider
-}
-
-func (b *BlockList) ShouldAllow(s string) bool {
-	b.mu.RLock()
-	d := b.Decider
-	b.mu.RUnlock()
-	if d == nil {
-		return true
-	}
-	return d(s)
-}
-
-// SetDecider atomically swaps the blocklist's decider. This method is
-// thread-safe.
-func (b *BlockList) SetDecider(d Decider) {
-	b.mu.Lock()
-	b.Decider = d
-	b.mu.Unlock()
-}
-
-func (b *BlockList) ShouldBlock(s string) bool {
-	return !b.ShouldAllow(s)
-}

core/corehttp/gateway_handler.go

@@ -84,6 +84,36 @@ func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	log.Error(errmsg) // TODO(cryptix): log errors until we have a better way to expose these (counter metrics maybe)
 }
 
+func (i *gatewayHandler) allowListBlocks(path string) bool {
+	if i.config.AllowList == nil {
+		return false
+	}
+
+	parts := strings.Split(path, "/")
+	if len(parts) < 3 {
+		return true
+	}
+
+	k := key.B58KeyDecode(parts[2])
+	if i.config.AllowList.Has(k) {
+		return false
+	}
+
+	return true
+}
+
+func (i *gatewayHandler) denyListBlocks(k key.Key) bool {
+	if i.config.DenyList == nil {
+		return false
+	}
+
+	if i.config.DenyList.Has(k) {
+		return true
+	}
+
+	return false
+}
+
 func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithCancel(i.node.Context())
 	defer cancel()
@@ -103,15 +133,28 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		ipnsHostname = true
 	}
 
-	if i.config.BlockList != nil && i.config.BlockList.ShouldBlock(urlPath) {
-		w.WriteHeader(http.StatusForbidden)
-		w.Write([]byte("403 - Forbidden"))
+	nd, err := core.Resolve(ctx, i.node, path.Path(urlPath))
+	if err != nil {
+		webError(w, "Path Resolve error", err, http.StatusBadRequest)
 		return
 	}
 
-	nd, err := core.Resolve(ctx, i.node, path.Path(urlPath))
+	k, err := nd.Key()
 	if err != nil {
-		webError(w, "Path Resolve error", err, http.StatusBadRequest)
+		log.Error("failed to get key from node: %s", err)
+		webError(w, "Marshaling Error", err, http.StatusInternalServerError)
+		return
+	}
+
+	if i.denyListBlocks(k) {
+		w.WriteHeader(http.StatusForbidden)
+		w.Write([]byte("403 - Forbidden (content on denylist)"))
+		return
+	}
+
+	if i.allowListBlocks(urlPath) {
+		w.WriteHeader(http.StatusForbidden)
+		w.Write([]byte("403 - Forbidden (content not on allowlist)"))
 		return
 	}
 

repo/config/gateway.go

@@ -5,4 +5,7 @@ type Gateway struct {
 	HTTPHeaders  map[string][]string // HTTP headers to return with the gateway
 	RootRedirect string
 	Writable     bool
+
+	DenyList  []string
+	AllowList []string
 }