feat: update dependencies, add middleware, pprof, fixed metrics

.deepsource.toml

@@ -1,9 +1,13 @@
 version = 1
 
+test_patterns = [
+  "**/*_test.go"
+]
+
 [[analyzers]]
 name = "go"
 enabled = true
 
   [analyzers.meta]
   import_root = "github.com/plexsysio/go-msuite"
-  dependencies_vendored = false
+  dependencies_vendored = false

core/core.go

@@ -3,21 +3,22 @@ package core
 
 import (
 	"context"
-	"github.com/SWRMLabs/ss-store"
+	"net/http"
+	"os"
+
 	ipfslite "github.com/hsanjuan/ipfs-lite"
 	"github.com/libp2p/go-libp2p-core/discovery"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/routing"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/plexsysio/dLocker"
+	store "github.com/plexsysio/gkvstore"
 	"github.com/plexsysio/go-msuite/modules/auth"
 	"github.com/plexsysio/go-msuite/modules/events"
 	"github.com/plexsysio/go-msuite/modules/repo"
 	"github.com/plexsysio/go-msuite/modules/sharedStorage"
 	"github.com/plexsysio/taskmanager"
 	"google.golang.org/grpc"
-	"net/http"
-	"os"
 )
 
 type Service interface {

go.mod

@@ -3,52 +3,47 @@ module github.com/plexsysio/go-msuite
 go 1.13
 
 require (
-	github.com/SWRMLabs/ants-db v0.0.3
-	github.com/SWRMLabs/ss-ds-store v0.0.7
-	github.com/SWRMLabs/ss-store v0.0.4
-	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/golang-jwt/jwt v3.2.1+incompatible
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hsanjuan/ipfs-lite v1.1.18
-	github.com/ipfs/go-blockservice v0.1.4
-	github.com/ipfs/go-cid v0.0.7
-	github.com/ipfs/go-datastore v0.4.5
-	github.com/ipfs/go-ds-flatfs v0.4.5
-	github.com/ipfs/go-ds-leveldb v0.4.2
-	github.com/ipfs/go-ipfs-blockstore v1.0.3
+	github.com/ipfs/go-blockservice v0.2.1
+	github.com/ipfs/go-cid v0.1.0
+	github.com/ipfs/go-datastore v0.5.0
+	github.com/ipfs/go-ds-flatfs v0.5.0
+	github.com/ipfs/go-ds-leveldb v0.5.0
+	github.com/ipfs/go-ipfs-blockstore v1.1.1
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/ipfs/go-log/v2 v2.3.0
-	github.com/ipfs/go-merkledag v0.3.2
-	github.com/libp2p/go-libp2p v0.13.0
+	github.com/ipfs/go-merkledag v0.5.1
+	github.com/libp2p/go-libp2p v0.16.0
 	github.com/libp2p/go-libp2p-connmgr v0.2.4
-	github.com/libp2p/go-libp2p-core v0.8.5
-	github.com/libp2p/go-libp2p-discovery v0.5.0
+	github.com/libp2p/go-libp2p-core v0.11.0
+	github.com/libp2p/go-libp2p-discovery v0.6.0
 	github.com/libp2p/go-libp2p-gostream v0.3.1
-	github.com/libp2p/go-libp2p-pubsub v0.4.1
-	github.com/libp2p/go-libp2p-quic-transport v0.10.0
-	github.com/libp2p/go-libp2p-tls v0.1.3
+	github.com/libp2p/go-libp2p-pubsub v0.6.0
+	github.com/libp2p/go-libp2p-quic-transport v0.15.0
+	github.com/libp2p/go-libp2p-tls v0.3.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moxiaomomo/grpc-jaeger v0.0.0-20180617090213-05b879580c4a
-	github.com/multiformats/go-multiaddr v0.3.1
+	github.com/multiformats/go-multiaddr v0.4.1
+	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
-	github.com/plexsysio/dLocker v0.0.2
+	github.com/plexsysio/ants-db v0.0.5-0.20211128140316-e6220c9cd405
+	github.com/plexsysio/dLocker v0.0.3-0.20211128192511-fcfcafab3562
+	github.com/plexsysio/gkvstore v0.0.0-20211118085618-aa2812d0ec8d
+	github.com/plexsysio/gkvstore-ipfsds v0.0.0-20211128070946-4089eab5b669
 	github.com/plexsysio/taskmanager v0.0.0-20210719193446-5b3bff8bc055
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rs/cors v1.7.0
 	github.com/slok/go-http-metrics v0.9.0
 	github.com/uber/jaeger-client-go v2.22.1+incompatible // indirect
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
 	go.uber.org/fx v1.10.0
-	go.uber.org/multierr v1.7.0 // indirect
-	go.uber.org/zap v1.18.1 // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
-	google.golang.org/grpc v1.36.0
+	google.golang.org/grpc v1.42.0
 )
 
-replace github.com/hsanjuan/ipfs-lite => github.com/plexsysio/ipfs-lite v1.1.21
-
-replace github.com/SWRMLabs/ants-db => github.com/plexsysio/ants-db v0.0.4
+replace github.com/hsanjuan/ipfs-lite => github.com/plexsysio/ipfs-lite v1.1.22-0.20211128135214-3d31c70fbf56

modules/auth/aclManager.go

@@ -1,9 +1,11 @@
 package auth
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
-	"github.com/SWRMLabs/ss-store"
+
+	store "github.com/plexsysio/gkvstore"
 	"github.com/plexsysio/go-msuite/modules/repo"
 )
 
@@ -57,11 +59,11 @@ type Acl struct {
 	Roles int
 }
 
-func (m *Acl) GetId() string {
+func (m *Acl) GetID() string {
 	return m.Key
 }
 
-func (m *Acl) GetNamespace() string {
+func (*Acl) GetNamespace() string {
 	return "Acl"
 }
 
@@ -100,21 +102,21 @@ func (a *aclManager) Configure(rsc string, role Role) error {
 		Key:   rsc,
 		Roles: r,
 	}
-	return a.st.Update(nacl)
+	return a.st.Update(context.TODO(), nacl)
 }
 
 func (a *aclManager) Delete(rsc string) error {
 	nacl := &Acl{
 		Key: rsc,
 	}
-	return a.st.Delete(nacl)
+	return a.st.Delete(context.TODO(), nacl)
 }
 
 func (a *aclManager) Authorized(rsc string, role Role) bool {
 	nacl := &Acl{
 		Key: rsc,
 	}
-	err := a.st.Read(nacl)
+	err := a.st.Read(context.TODO(), nacl)
 	if err != nil {
 		// If there is no ACL configured, by default access is universal
 		return true
@@ -130,7 +132,7 @@ func (a *aclManager) Allowed(rsc string) []Role {
 	nacl := &Acl{
 		Key: rsc,
 	}
-	err := a.st.Read(nacl)
+	err := a.st.Read(context.TODO(), nacl)
 	if err != nil {
 		// If there is no ACL configured, by default access is universal
 		return []Role{None, PublicRead, PublicWrite, AuthRead, AuthWrite, Admin}

modules/auth/aclManager_test.go

@@ -3,13 +3,13 @@ package auth_test
 import (
 	"testing"
 
-	dsstore "github.com/SWRMLabs/ss-ds-store"
-	"github.com/SWRMLabs/ss-store"
 	ds "github.com/ipfs/go-datastore"
 	syncds "github.com/ipfs/go-datastore/sync"
+	store "github.com/plexsysio/gkvstore"
+	dsstore "github.com/plexsysio/gkvstore-ipfsds"
 	"github.com/plexsysio/go-msuite/modules/auth"
 	"github.com/plexsysio/go-msuite/modules/config"
-	"github.com/plexsysio/go-msuite/modules/config/json"
+	jsonConf "github.com/plexsysio/go-msuite/modules/config/json"
 )
 
 type testRepo struct {
@@ -21,7 +21,7 @@ type testRepo struct {
 func newTestRepo() *testRepo {
 	cfg := jsonConf.DefaultConfig()
 	bs := syncds.MutexWrap(ds.NewMapDatastore())
-	st, _ := dsstore.NewDataStore(&dsstore.DSConfig{DS: bs})
+	st := dsstore.New(bs)
 	return &testRepo{st: st, cfg: cfg, d: bs}
 }
 

modules/diag/metrics/metrics.go

@@ -0,0 +1,50 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+
+	gtrace "github.com/moxiaomomo/grpc-jaeger"
+	"github.com/opentracing/opentracing-go"
+	"github.com/plexsysio/go-msuite/modules/config"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"go.uber.org/fx"
+)
+
+func New() *prometheus.Registry {
+	r := prometheus.NewRegistry()
+
+	r.MustRegister(
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{
+			Namespace: "msuite",
+		}),
+		collectors.NewGoCollector(),
+	)
+
+	return r
+}
+
+func NewTracer(lc fx.Lifecycle, conf config.Config) (opentracing.Tracer, error) {
+	svcName := "default"
+	conf.Get("TracingName", &svcName)
+
+	var tHost string
+	ok := conf.Get("TracingHost", &tHost)
+	if !ok {
+		return nil, errors.New("Tracing host not specified")
+	}
+
+	tracer, closer, err := gtrace.NewJaegerTracer(svcName, tHost)
+	if err != nil {
+		return nil, err
+	}
+
+	lc.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			return closer.Close()
+		},
+	})
+
+	return tracer, nil
+}

modules/grpc/mux/mux.go

@@ -40,7 +40,7 @@ func New(
 	return m
 }
 
-func (m *Mux) Start(reportError func(string, error)) {
+func (m *Mux) Start(ctx context.Context, reportError func(string, error)) {
 	for _, v := range m.listeners {
 		l := &muxListener{
 			tag:      v.Tag,
@@ -52,10 +52,15 @@ func (m *Mux) Start(reportError func(string, error)) {
 				}
 			},
 		}
-		_, err := m.tm.Go(l)
+		sched, err := m.tm.Go(l)
 		if err != nil {
 			reportError(v.Tag, err)
 		}
+		select {
+		case <-sched:
+		case <-ctx.Done():
+			return
+		}
 	}
 }
 

modules/grpc/mux/mux_test.go

@@ -7,14 +7,27 @@ import (
 	"testing"
 	"time"
 
-	"github.com/plexsysio/go-msuite/modules/grpc/mux"
+	logger "github.com/ipfs/go-log/v2"
+	grpcmux "github.com/plexsysio/go-msuite/modules/grpc/mux"
 	"github.com/plexsysio/taskmanager"
 )
 
 func TestMultipleListeners(t *testing.T) {
-	tcpListener1, _ := net.Listen("tcp", ":8080")
-	tcpListener2, _ := net.Listen("tcp", ":8081")
-	tcpListener3, _ := net.Listen("tcp", ":8082")
+	_ = logger.SetLogLevel("grpc/lmux", "*")
+
+	tcpListener1, err := net.Listen("tcp", ":10081")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tcpListener2, err := net.Listen("tcp", ":10082")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tcpListener3, err := net.Listen("tcp", ":10083")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	tm := taskmanager.New(4, 10, time.Second*10)
 
 	listeners := []grpcmux.MuxListener{
@@ -38,7 +51,7 @@ func TestMultipleListeners(t *testing.T) {
 		tm,
 	)
 
-	m.Start(nil)
+	m.Start(context.TODO(), nil)
 
 	connChan := make(chan net.Conn)
 
@@ -76,15 +89,15 @@ func TestMultipleListeners(t *testing.T) {
 		close(done)
 	}()
 
-	_, err := net.Dial("tcp", ":8080")
+	_, err = net.Dial("tcp", ":10081")
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = net.Dial("tcp", ":8081")
+	_, err = net.Dial("tcp", ":10082")
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = net.Dial("tcp", ":8082")
+	_, err = net.Dial("tcp", ":10083")
 	if err != nil {
 		t.Fatal(err)
 	}