Skip to content

Commit

Permalink
swarm: initial instrumentation (ethereum#15969)
Browse files Browse the repository at this point in the history 
* swarm: initial instrumentation with go-metrics

* swarm: initialise metrics collection and add ResettingTimer to HTTP requests

* swarm: update metrics flags names. remove redundant Timer.

* swarm: rename method for periodically updating gauges

* swarm: finalise metrics after feedback

* swarm/network: always init kad metrics containers

* swarm/network: off-by-one index in metrics containers

* swarm, metrics: resolved conflicts
  • Loading branch information
@nonsense @mariameda
nonsense authored and mariameda committed Aug 19, 2018
1 parent 34e2509 commit fe636d7
Show file tree
Hide file tree
Showing 16 changed files with 434 additions and 65 deletions.
8 changes: 7 additions & 1 deletion cmd/swarm/main.go
View file
Expand Up @@ -48,6 +48,7 @@ import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm"
bzzapi "github.com/ethereum/go-ethereum/swarm/api"
swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"

"gopkg.in/urfave/cli.v1"
)
Expand Down Expand Up @@ -365,9 +366,14 @@ DEPRECATED: use 'swarm db clean'.
DeprecatedEthAPIFlag,
}
app.Flags = append(app.Flags, debug.Flags...)
app.Flags = append(app.Flags, swarmmetrics.Flags...)
app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
return debug.Setup(ctx)
if err := debug.Setup(ctx); err != nil {
return err
}
swarmmetrics.Setup(ctx)
return nil
}
app.After = func(ctx *cli.Context) error {
debug.Exit()
Expand Down
1 change: 0 additions & 1 deletion metrics/metrics.go
View file
Expand Up @@ -35,7 +35,6 @@ func init() {
Enabled = true
}
}
//exp.Exp(DefaultRegistry)
}

// CollectProcessMetrics periodically collects various metrics about the running
Expand Down
69 changes: 61 additions & 8 deletions swarm/api/api.go 100755 → 100644
View file
@@ -1,18 +1,18 @@
// Copyright 2016 The go-nilu Authors
// This file is part of the go-nilu library.
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-nilu library is free software: you can redistribute it and/or modify
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-nilu library is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-nilu library. If not, see <http://www.gnu.org/licenses/>.
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package api

Expand All @@ -30,13 +30,33 @@ import (
"path/filepath"
"time"

"github.com/NiluPlatform/go-nilu/common"
"github.com/NiluPlatform/go-nilu/log"
"github.com/NiluPlatform/go-nilu/swarm/storage"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/storage"
)

var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}")

//setup metrics
var (
apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil)
apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil)
apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil)
apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil)
apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil)
apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil)
apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil)
apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil)
apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil)
apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil)
apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil)
apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil)
apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil)
apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil)
apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil)
)

type Resolver interface {
Resolve(string) (common.Hash, error)
}
Expand Down Expand Up @@ -155,6 +175,7 @@ type ErrResolve error

// DNS Resolver
func (self *Api) Resolve(uri *URI) (storage.Key, error) {
apiResolveCount.Inc(1)
log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))

// if the URI is immutable, check if the address is a hash
Expand All @@ -169,6 +190,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
// if DNS is not configured, check if the address is a hash
if self.dns == nil {
if !isHash {
apiResolveFail.Inc(1)
return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
}
return common.Hex2Bytes(uri.Addr), nil
Expand All @@ -179,23 +201,27 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
if err == nil {
return resolved[:], nil
} else if !isHash {
apiResolveFail.Inc(1)
return nil, err
}
return common.Hex2Bytes(uri.Addr), nil
}

// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (storage.Key, error) {
apiPutCount.Inc(1)
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
}
wg.Wait()
Expand All @@ -206,8 +232,10 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) {
// to resolve basePath to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
apiGetCount.Inc(1)
trie, err := loadManifest(self.dpa, key, nil)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
return
Expand All @@ -221,6 +249,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe
key = common.Hex2Bytes(entry.Hash)
status = entry.Status
if status == http.StatusMultipleChoices {
apiGetHttp300.Inc(1)
return
} else {
mimeType = entry.ContentType
Expand All @@ -229,16 +258,19 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe
}
} else {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest entry for '%s' not found", path)
log.Warn(fmt.Sprintf("%v", err))
}
return
}

func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
apiModifyCount.Inc(1)
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
apiModifyFail.Inc(1)
return nil, err
}
if contentHash != "" {
Expand All @@ -253,19 +285,23 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string)
}

if err := trie.recalcAndStore(); err != nil {
apiModifyFail.Inc(1)
return nil, err
}
return trie.hash, nil
}

func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
apiAddFileCount.Inc(1)

uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}

Expand All @@ -284,16 +320,19 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver

mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}

fkey, err := mw.AddEntry(bytes.NewReader(content), entry)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}

newMkey, err := mw.Store()
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err

}
Expand All @@ -303,13 +342,16 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver
}

func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
apiRmFileCount.Inc(1)

uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}

Expand All @@ -320,16 +362,19 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin

mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}

err = mw.RemoveEntry(filepath.Join(path, fname))
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}

newMkey, err := mw.Store()
if err != nil {
apiRmFileFail.Inc(1)
return "", err

}
Expand All @@ -338,6 +383,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin
}

func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) {
apiAppendFileCount.Inc(1)

buffSize := offset + addSize
if buffSize < existingSize {
Expand Down Expand Up @@ -366,10 +412,12 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte

uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}

Expand All @@ -380,11 +428,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte

mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}

err = mw.RemoveEntry(filepath.Join(path, fname))
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}

Expand All @@ -398,11 +448,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte

fkey, err := mw.AddEntry(io.Reader(combinedReader), entry)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}

newMkey, err := mw.Store()
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err

}
Expand All @@ -412,6 +464,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
}

func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) {

uri, err := Parse("bzz:/" + mhash)
if err != nil {
return nil, nil, err
Expand Down
23 changes: 16 additions & 7 deletions swarm/api/http/error.go 100755 → 100644
View file
@@ -1,18 +1,18 @@
// Copyright 2017 The go-nilu Authors
// This file is part of the go-nilu library.
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-nilu library is free software: you can redistribute it and/or modify
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-nilu library is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-nilu library. If not, see <http://www.gnu.org/licenses/>.
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

/*
Show nicely (but simple) formatted HTML error pages (or respond with JSON
Expand All @@ -28,13 +28,20 @@ import (
"strings"
"time"

"github.com/NiluPlatform/go-nilu/log"
"github.com/NiluPlatform/go-nilu/swarm/api"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
)

//templateMap holds a mapping of an HTTP error code to a template
var templateMap map[int]*template.Template

//metrics variables
var (
htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
)

//parameters needed for formatting the correct HTML page
type ErrorParams struct {
Msg string
Expand Down Expand Up @@ -132,6 +139,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) {

//return a HTML page
func respondHtml(w http.ResponseWriter, params *ErrorParams) {
htmlCounter.Inc(1)
err := params.template.Execute(w, params)
if err != nil {
log.Error(err.Error())
Expand All @@ -140,6 +148,7 @@ func respondHtml(w http.ResponseWriter, params *ErrorParams) {

//return JSON
func respondJson(w http.ResponseWriter, params *ErrorParams) {
jsonCounter.Inc(1)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(params)
}
Expand Down

0 comments on commit fe636d7

Please sign in to comment.