From 77f6b8726d492e17af7c2d60e3757b48da6269cb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 28 Jul 2015 18:10:11 -0700 Subject: [PATCH 1/3] implement access control lists for gateway License: MIT Signed-off-by: Jeromy --- blocks/key/key_set.go | 74 ++++++++++++++++++++++--------- cmd/ipfs/daemon.go | 75 ++++++++++++++++++++++++-------- core/corehttp/gateway.go | 41 +++-------------- core/corehttp/gateway_handler.go | 23 +++++++--- repo/config/gateway.go | 6 +++ 5 files changed, 141 insertions(+), 78 deletions(-) diff --git a/blocks/key/key_set.go b/blocks/key/key_set.go index f9e177d6a3b..807cda68350 100644 --- a/blocks/key/key_set.go +++ b/blocks/key/key_set.go @@ -6,41 +6,73 @@ import ( type KeySet interface { Add(Key) + Has(Key) bool Remove(Key) Keys() []Key } -type ks struct { - lock sync.RWMutex - data map[Key]struct{} +type keySet struct { + keys map[Key]struct{} } -func NewKeySet() KeySet { - return &ks{ - data: make(map[Key]struct{}), +func NewKeySet() *keySet { + return &keySet{make(map[Key]struct{})} +} + +func (gcs *keySet) Add(k Key) { + gcs.keys[k] = struct{}{} +} + +func (gcs *keySet) Has(k Key) bool { + _, has := gcs.keys[k] + return has +} + +func (ks *keySet) Keys() []Key { + var out []Key + for k, _ := range ks.keys { + out = append(out, k) } + return out +} + +func (ks *keySet) Remove(k Key) { + delete(ks.keys, k) } -func (wl *ks) Add(k Key) { - wl.lock.Lock() - defer wl.lock.Unlock() +// TODO: implement disk-backed keyset for working with massive DAGs - wl.data[k] = struct{}{} +type threadsafe struct { + lk sync.Mutex + ks KeySet } -func (wl *ks) Remove(k Key) { - wl.lock.Lock() - defer wl.lock.Unlock() +func Threadsafe(ks KeySet) *threadsafe { + return &threadsafe{ks: ks} +} - delete(wl.data, k) +func (ts *threadsafe) Has(k Key) bool { + ts.lk.Lock() + out := ts.ks.Has(k) + ts.lk.Unlock() //defer is slow + return out } -func (wl *ks) Keys() []Key { - wl.lock.RLock() - defer wl.lock.RUnlock() - keys := make([]Key, 0) - for k := range wl.data { - keys = append(keys, k) - } +func (ts *threadsafe) Remove(k Key) { + ts.lk.Lock() + ts.ks.Remove(k) + ts.lk.Unlock() //defer is slow +} + +func (ts *threadsafe) Add(k Key) { + ts.lk.Lock() + ts.ks.Add(k) + ts.lk.Unlock() //defer is slow +} + +func (ts *threadsafe) Keys() []Key { + ts.lk.Lock() + keys := ts.ks.Keys() + ts.lk.Unlock() //defer is slow return keys } diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 9a44c960b6b..6f638ce5b72 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -1,6 +1,7 @@ package main import ( + "bufio" _ "expvar" "fmt" "net" @@ -8,13 +9,13 @@ import ( _ "net/http/pprof" "os" "sort" - "strings" "sync" _ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/codahale/metrics/runtime" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + key "github.com/ipfs/go-ipfs/blocks/key" cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" commands "github.com/ipfs/go-ipfs/core/commands" @@ -310,23 +311,20 @@ func serveHTTPApi(req cmds.Request) (error, <-chan error) { return fmt.Errorf("serveHTTPApi: Option(%s) failed: %s", unrestrictedApiAccessKwd, err), nil } + var whitelist key.KeySet + if !unrestricted { + whitelist = key.Threadsafe(key.NewKeySet()) + for _, webuipath := range corehttp.WebUIPaths { + // extract the key + whitelist.Add(key.B58KeyDecode(webuipath[6:])) + } + } + apiGw := corehttp.NewGateway(corehttp.GatewayConfig{ - Writable: true, - BlockList: &corehttp.BlockList{ - Decider: func(s string) bool { - if unrestricted { - return true - } - // for now, only allow paths in the WebUI path - for _, webuipath := range corehttp.WebUIPaths { - if strings.HasPrefix(s, webuipath) { - return true - } - } - return false - }, - }, + Writable: true, + WhiteList: whitelist, }) + var opts = []corehttp.ServeOption{ corehttp.CommandsOption(*req.InvocContext()), corehttp.WebUIOption, @@ -401,11 +399,36 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) { fmt.Printf("Gateway (readonly) server listening on %s\n", gatewayMaddr) } + var blacklist key.KeySet + var whitelist key.KeySet + if cfg.Gateway.BlackList != "" { + l, err := loadKeySetFromURL(cfg.Gateway.BlackList) + if err != nil { + return err, nil + } + blacklist = l + } + + if cfg.Gateway.WhiteList != "" { + l, err := loadKeySetFromURL(cfg.Gateway.WhiteList) + if err != nil { + return err, nil + } + whitelist = l + } + + gateway := corehttp.Gateway{ + Config: corehttp.GatewayConfig{ + BlackList: blacklist, + WhiteList: whitelist, + }, + } + var opts = []corehttp.ServeOption{ corehttp.CommandsROOption(*req.InvocContext()), corehttp.VersionOption(), corehttp.IPNSHostnameOption(), - corehttp.GatewayOption(writable), + gateway.ServeOption(), } if len(cfg.Gateway.RootRedirect) > 0 { @@ -425,6 +448,24 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) { return nil, errc } +func loadKeySetFromURL(url string) (key.KeySet, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + ks := key.NewKeySet() + scan := bufio.NewScanner(resp.Body) + for scan.Scan() { + k := key.B58KeyDecode(scan.Text()) + if k == "" { + return nil, fmt.Errorf("invalid key in set") + } + ks.Add(k) + } + return key.Threadsafe(ks), nil +} + //collects options and opens the fuse mountpoint func mountFuse(req cmds.Request) error { cfg, err := req.InvocContext().GetConfig() diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 584a894376a..e145fde174a 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -4,8 +4,8 @@ import ( "fmt" "net" "net/http" - "sync" + key "github.com/ipfs/go-ipfs/blocks/key" core "github.com/ipfs/go-ipfs/core" id "github.com/ipfs/go-ipfs/p2p/protocol/identify" ) @@ -17,8 +17,10 @@ type Gateway struct { type GatewayConfig struct { Headers map[string][]string - BlockList *BlockList - Writable bool + BlackList key.KeySet + WhiteList key.KeySet + + Writable bool } func NewGateway(conf GatewayConfig) *Gateway { @@ -44,8 +46,7 @@ func (g *Gateway) ServeOption() ServeOption { func GatewayOption(writable bool) ServeOption { g := NewGateway(GatewayConfig{ - Writable: writable, - BlockList: &BlockList{}, + Writable: writable, }) return g.ServeOption() } @@ -59,33 +60,3 @@ func VersionOption() ServeOption { return mux, nil } } - -// Decider decides whether to Allow string -type Decider func(string) bool - -type BlockList struct { - mu sync.RWMutex - Decider Decider -} - -func (b *BlockList) ShouldAllow(s string) bool { - b.mu.RLock() - d := b.Decider - b.mu.RUnlock() - if d == nil { - return true - } - return d(s) -} - -// SetDecider atomically swaps the blocklist's decider. This method is -// thread-safe. -func (b *BlockList) SetDecider(d Decider) { - b.mu.Lock() - b.Decider = d - b.mu.Unlock() -} - -func (b *BlockList) ShouldBlock(s string) bool { - return !b.ShouldAllow(s) -} diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 154db3b5307..2d841003d0b 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -103,15 +103,28 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request ipnsHostname = true } - if i.config.BlockList != nil && i.config.BlockList.ShouldBlock(urlPath) { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("403 - Forbidden")) + nd, err := core.Resolve(ctx, i.node, path.Path(urlPath)) + if err != nil { + webError(w, "Path Resolve error", err, http.StatusBadRequest) return } - nd, err := core.Resolve(ctx, i.node, path.Path(urlPath)) + k, err := nd.Key() if err != nil { - webError(w, "Path Resolve error", err, http.StatusBadRequest) + log.Error("failed to get key from node: %s", err) + webError(w, "Marshaling Error", err, http.StatusInternalServerError) + return + } + + if i.config.BlackList != nil && i.config.BlackList.Has(k) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("403 - Forbidden (content on blacklist)")) + return + } + + if i.config.WhiteList != nil && !i.config.WhiteList.Has(k) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("403 - Forbidden (content on blacklist)")) return } diff --git a/repo/config/gateway.go b/repo/config/gateway.go index 07bc9aad2cb..cbc2147dd6d 100644 --- a/repo/config/gateway.go +++ b/repo/config/gateway.go @@ -5,4 +5,10 @@ type Gateway struct { HTTPHeaders map[string][]string // HTTP headers to return with the gateway RootRedirect string Writable bool + + // The url of a newline delimited list of keys that the gateway should not serve + BlackList string + + // The url of a newline delimited list of keys that the gateway should only serve + WhiteList string } From 9be3b2cfd3feb2bdac8539343e27e57801180221 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 3 Aug 2015 13:45:49 -0700 Subject: [PATCH 2/3] change naming and parse format License: MIT Signed-off-by: Jeromy --- cmd/ipfs/daemon.go | 63 +++++++++++++++++++------------- core/corehttp/gateway.go | 4 +- core/corehttp/gateway_handler.go | 10 ++--- repo/config/gateway.go | 7 +--- 4 files changed, 46 insertions(+), 38 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 6f638ce5b72..979996a460b 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -1,7 +1,7 @@ package main import ( - "bufio" + "encoding/json" _ "expvar" "fmt" "net" @@ -311,18 +311,18 @@ func serveHTTPApi(req cmds.Request) (error, <-chan error) { return fmt.Errorf("serveHTTPApi: Option(%s) failed: %s", unrestrictedApiAccessKwd, err), nil } - var whitelist key.KeySet + var allowlist key.KeySet if !unrestricted { - whitelist = key.Threadsafe(key.NewKeySet()) + allowlist = key.Threadsafe(key.NewKeySet()) for _, webuipath := range corehttp.WebUIPaths { // extract the key - whitelist.Add(key.B58KeyDecode(webuipath[6:])) + allowlist.Add(key.B58KeyDecode(webuipath[6:])) } } apiGw := corehttp.NewGateway(corehttp.GatewayConfig{ Writable: true, - WhiteList: whitelist, + AllowList: allowlist, }) var opts = []corehttp.ServeOption{ @@ -399,28 +399,28 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) { fmt.Printf("Gateway (readonly) server listening on %s\n", gatewayMaddr) } - var blacklist key.KeySet - var whitelist key.KeySet - if cfg.Gateway.BlackList != "" { - l, err := loadKeySetFromURL(cfg.Gateway.BlackList) + var denylist key.KeySet + var allowlist key.KeySet + if len(cfg.Gateway.DenyList) > 0 { + l, err := loadKeySetFromURLs(cfg.Gateway.DenyList) if err != nil { return err, nil } - blacklist = l + denylist = l } - if cfg.Gateway.WhiteList != "" { - l, err := loadKeySetFromURL(cfg.Gateway.WhiteList) + if len(cfg.Gateway.AllowList) > 0 { + l, err := loadKeySetFromURLs(cfg.Gateway.AllowList) if err != nil { return err, nil } - whitelist = l + allowlist = l } gateway := corehttp.Gateway{ Config: corehttp.GatewayConfig{ - BlackList: blacklist, - WhiteList: whitelist, + DenyList: denylist, + AllowList: allowlist, }, } @@ -448,20 +448,31 @@ func serveHTTPGateway(req cmds.Request) (error, <-chan error) { return nil, errc } -func loadKeySetFromURL(url string) (key.KeySet, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } +type listing struct { + Uri string + Keys []string +} +func loadKeySetFromURLs(urls []string) (key.KeySet, error) { ks := key.NewKeySet() - scan := bufio.NewScanner(resp.Body) - for scan.Scan() { - k := key.B58KeyDecode(scan.Text()) - if k == "" { - return nil, fmt.Errorf("invalid key in set") + for _, url := range urls { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + var list listing + err = json.NewDecoder(resp.Body).Decode(&list) + if err != nil { + return nil, err + } + for _, k := range list.Keys { + lk := key.B58KeyDecode(k) + if lk == "" { + return nil, fmt.Errorf("incorrectly formatted key '%s'", k) + } + ks.Add(lk) } - ks.Add(k) } return key.Threadsafe(ks), nil } diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index e145fde174a..046c0478a0e 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -17,8 +17,8 @@ type Gateway struct { type GatewayConfig struct { Headers map[string][]string - BlackList key.KeySet - WhiteList key.KeySet + DenyList key.KeySet + AllowList key.KeySet Writable bool } diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 2d841003d0b..a365b843c73 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -116,15 +116,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return } - if i.config.BlackList != nil && i.config.BlackList.Has(k) { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("403 - Forbidden (content on blacklist)")) + if i.config.DenyList != nil && i.config.DenyList.Has(k) { + w.WriteHeader(451) + w.Write([]byte("451 - Unavailable For Legal Reasons")) return } - if i.config.WhiteList != nil && !i.config.WhiteList.Has(k) { + if i.config.AllowList != nil && !i.config.AllowList.Has(k) { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("403 - Forbidden (content on blacklist)")) + w.Write([]byte("403 - Forbidden (content not on whitelist)")) return } diff --git a/repo/config/gateway.go b/repo/config/gateway.go index cbc2147dd6d..1973170ae10 100644 --- a/repo/config/gateway.go +++ b/repo/config/gateway.go @@ -6,9 +6,6 @@ type Gateway struct { RootRedirect string Writable bool - // The url of a newline delimited list of keys that the gateway should not serve - BlackList string - - // The url of a newline delimited list of keys that the gateway should only serve - WhiteList string + DenyList []string + AllowList []string } From 13374c374600cf977a22f76eb06252926ec9d346 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 25 Aug 2015 14:45:27 -0700 Subject: [PATCH 3/3] correctly filter content based on lists License: MIT Signed-off-by: Jeromy --- core/corehttp/gateway_handler.go | 40 ++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index a365b843c73..29e4a60877c 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -84,6 +84,36 @@ func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Error(errmsg) // TODO(cryptix): log errors until we have a better way to expose these (counter metrics maybe) } +func (i *gatewayHandler) allowListBlocks(path string) bool { + if i.config.AllowList == nil { + return false + } + + parts := strings.Split(path, "/") + if len(parts) < 3 { + return true + } + + k := key.B58KeyDecode(parts[2]) + if i.config.AllowList.Has(k) { + return false + } + + return true +} + +func (i *gatewayHandler) denyListBlocks(k key.Key) bool { + if i.config.DenyList == nil { + return false + } + + if i.config.DenyList.Has(k) { + return true + } + + return false +} + func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithCancel(i.node.Context()) defer cancel() @@ -116,15 +146,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return } - if i.config.DenyList != nil && i.config.DenyList.Has(k) { - w.WriteHeader(451) - w.Write([]byte("451 - Unavailable For Legal Reasons")) + if i.denyListBlocks(k) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("403 - Forbidden (content on denylist)")) return } - if i.config.AllowList != nil && !i.config.AllowList.Has(k) { + if i.allowListBlocks(urlPath) { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("403 - Forbidden (content not on whitelist)")) + w.Write([]byte("403 - Forbidden (content not on allowlist)")) return }