Merge branch 'main' into add-understanding-ferretdb-documentation-1215

.gitignore

@@ -26,7 +26,7 @@ node_modules/
 state.json
 
 # for now
-testdata/fuzz/
+**/testdata/fuzz/
 records/
 
 # for local Docker Compse changes; see https://docs.docker.com/compose/extends/#multiple-compose-files

.golangci.yml

@@ -70,6 +70,8 @@ linters-settings:
     alias:
       - pkg: github.com/tigrisdata/tigris-client-go/api/server/v1
         alias: api
+      - pkg: github.com/prometheus/client_model/go
+        alias: dto
   # ineffassign
   lll:
     line-length: 130
@@ -132,10 +134,13 @@ issues:
       path: internal/wire
       text: bson
 
-    # only `testutil` and `wire` packages can import `fjson` package
+    # only `testutil`, `bson` (for testing), and `wire` packages can import `fjson` package
     - linters: [depguard]
       path: internal/util/testutil
       text: fjson
+    - linters: [depguard]
+      path: internal/bson
+      text: fjson
     - linters: [depguard]
       path: internal/wire
       text: fjson

CONTRIBUTING.md

@@ -96,7 +96,7 @@ The `internal` subpackages contain most of the FerretDB code:
 * `types/fjson` provides converters from/to FJSON for built-in and `types` types.
   FJSON adds some extensions to JSON for keeping object keys in order,
   preserving BSON type information in the values themselves, etc.
-  It is used for logging.
+  It is used for logging of BSON values and wire protocol messages.
 * `bson` package provides converters from/to BSON for built-in and `types` types.
 * `wire` package provides wire protocol implementation.
 * `clientconn` package provides client connection implementation.

Taskfile.yml

@@ -174,17 +174,14 @@ tasks:
         -coverprofile=integration-mongodb.txt . -target-port=37017 -compat-port=0
 
   bench-short:
-    desc: "Benchmark for about 40 seconds (with default BENCHTIME)"
+    desc: "Benchmark for about 25 seconds (with default BENCHTIME)"
     cmds:
       - go test -list='Benchmark.*' ./...
-      - echo 'Running eight functions for {{.BENCHTIME}} each...'
-      - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/types/fjson/       | tee -a new.txt
-      - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/types/fjson/       | tee -a new.txt
+      - echo 'Running five functions for {{.BENCHTIME}} each...'
       - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/bson/              | tee -a new.txt
       - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/bson/              | tee -a new.txt
       - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/handlers/pg/pjson/ | tee -a new.txt
       - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/handlers/pg/pjson/ | tee -a new.txt
-      - go test -bench=BenchmarkArray    -benchtime={{.BENCHTIME}} ./internal/tjson/             | tee -a new.txt
       - go test -bench=BenchmarkDocument -benchtime={{.BENCHTIME}} ./internal/tjson/             | tee -a new.txt
       - bin/benchstat{{exeExt}} old.txt new.txt
 
@@ -198,17 +195,14 @@ tasks:
   # Those commands should still run tests (i.e., should not have -run=XXX flags)
   # to fill seed corpus for fuzz tests that use WriteSeedCorpusFile (e.g., FuzzHandler).
   fuzz:
-    desc: "Fuzz for about 3 minutes (with default FUZZTIME)"
+    desc: "Fuzz for about 2 minutes (with default FUZZTIME)"
     cmds:
       - go test -list='Fuzz.*' ./...
-      - echo 'Running eleven functions for {{.FUZZTIME}} each...'
-      - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/types/fjson/
-      - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/types/fjson/
+      - echo 'Running eight functions for {{.FUZZTIME}} each...'
       - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/bson/
       - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/bson/
       - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/handlers/pg/pjson/
       - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/handlers/pg/pjson/
-      - go test -fuzz=FuzzArray    -fuzztime={{.FUZZTIME}} ./internal/tjson/
       - go test -fuzz=FuzzDocument -fuzztime={{.FUZZTIME}} ./internal/tjson/
       - go test -fuzz=FuzzMsg      -fuzztime={{.FUZZTIME}} ./internal/wire/
       - go test -fuzz=FuzzQuery    -fuzztime={{.FUZZTIME}} ./internal/wire/

cmd/ferretdb/main.go

@@ -20,6 +20,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -28,8 +29,11 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+	"golang.org/x/exp/maps"
 
 	"github.com/FerretDB/FerretDB/internal/clientconn"
+	"github.com/FerretDB/FerretDB/internal/clientconn/connmetrics"
+	"github.com/FerretDB/FerretDB/internal/handlers/common"
 	"github.com/FerretDB/FerretDB/internal/handlers/registry"
 	"github.com/FerretDB/FerretDB/internal/util/debug"
 	"github.com/FerretDB/FerretDB/internal/util/logging"
@@ -214,9 +218,15 @@ func run() {
 
 	go debug.RunHandler(ctx, cli.DebugAddr, metricsRegisterer, logger.Named("debug"))
 
+	cmdsList := maps.Keys(common.Commands)
+	sort.Strings(cmdsList)
+
+	metrics := connmetrics.NewListenerMetrics(cmdsList)
+
 	h, err := registry.NewHandler(cli.Handler, &registry.NewHandlerOpts{
-		Ctx:    ctx,
-		Logger: logger,
+		Ctx:     ctx,
+		Logger:  logger,
+		Metrics: metrics.ConnMetrics,
 
 		PostgreSQLURL: cli.PostgreSQLURL,
 
@@ -235,6 +245,7 @@ func run() {
 		ProxyAddr:       cli.ProxyAddr,
 		Mode:            clientconn.Mode(cli.Mode),
 		Handler:         h,
+		Metrics:         metrics,
 		Logger:          logger,
 		TestConnTimeout: cli.Test.ConnTimeout,
 		TestRecordsDir:  cli.Test.RecordsDir,

cmd/fuzztool/fuzztool.go

@@ -227,19 +227,20 @@ func main() {
 		case "generated":
 			src = generatedCorpus
 		default:
-			src, err = filepath.Abs(src)
+			src, err = filepath.Abs(cli.Corpus.Src)
 			if err != nil {
 				logger.Fatal(err)
 			}
 		}
 
 		switch cli.Corpus.Dst {
 		case "seed":
-			dst = seedCorpus
+			// Because we would need to add `/testdata/fuzz` back, and that's not very easy.
+			logger.Fatal("Copying to seed corpus is not supported.")
 		case "generated":
 			dst = generatedCorpus
 		default:
-			dst, err = filepath.Abs(dst)
+			dst, err = filepath.Abs(cli.Corpus.Dst)
 			if err != nil {
 				logger.Fatal(err)
 			}

ferretdb/ferretdb.go

@@ -20,10 +20,14 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"sort"
 
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 
 	"github.com/FerretDB/FerretDB/internal/clientconn"
+	"github.com/FerretDB/FerretDB/internal/clientconn/connmetrics"
+	"github.com/FerretDB/FerretDB/internal/handlers/common"
 	"github.com/FerretDB/FerretDB/internal/handlers/registry"
 	"github.com/FerretDB/FerretDB/internal/util/logging"
 )
@@ -71,9 +75,15 @@ func New(config *Config) (*FerretDB, error) {
 //
 // When this method returns, listener and all connections are closed.
 func (f *FerretDB) Run(ctx context.Context) error {
+	cmdsList := maps.Keys(common.Commands)
+	sort.Strings(cmdsList)
+
+	metrics := connmetrics.NewListenerMetrics(cmdsList)
+
 	newOpts := registry.NewHandlerOpts{
-		Ctx:    context.Background(),
-		Logger: logger,
+		Ctx:     context.Background(),
+		Logger:  logger,
+		Metrics: metrics.ConnMetrics,
 
 		PostgreSQLURL: f.config.PostgreSQLURL,
 
@@ -93,6 +103,7 @@ func (f *FerretDB) Run(ctx context.Context) error {
 		Mode:       clientconn.NormalMode,
 		Handler:    h,
 		Logger:     logger,
+		Metrics:    metrics,
 	})
 
 	if err = l.Run(ctx); err != nil {

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/jackc/pgx/v4 v4.17.2
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.13.0
+	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.37.0
 	github.com/stretchr/testify v1.8.0
 	github.com/tigrisdata/tigris-client-go v1.0.0-beta.2
@@ -49,7 +50,6 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect

integration/.golangci.yml

@@ -55,6 +55,8 @@ linters-settings:
     alias:
       - pkg: github.com/tigrisdata/tigris-client-go/api/server/v1
         alias: api
+      - pkg: github.com/prometheus/client_model/go
+        alias: dto
   # ineffassign
   lll:
     line-length: 130

integration/commands_administration_test.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"math"
 	"net"
+	"sort"
 	"strconv"
 	"testing"
 	"time"
@@ -838,6 +839,86 @@ func TestCommandsAdministrationServerStatus(t *testing.T) {
 	assert.Equal(t, int32(0), must.NotFail(catalogStats.Get("internalViews")))
 }
 
+func TestCommandsAdministrationServerStatusMetrics(t *testing.T) {
+	t.Parallel()
+
+	for name, tc := range map[string]struct {
+		cmds           []bson.D
+		metricsPath    types.Path
+		expectedFields []string
+		expectedNoZero []string
+	}{
+		"BasicCmd": {
+			cmds: []bson.D{
+				{{"ping", int32(1)}},
+			},
+			metricsPath:    types.NewPath([]string{"metrics", "commands", "ping"}),
+			expectedFields: []string{"total", "failed"},
+			expectedNoZero: []string{"total"},
+		},
+		"UpdateCmd": {
+			cmds: []bson.D{
+				{{"update", "values"}, {"updates", bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}}}}},
+			},
+			metricsPath:    types.NewPath([]string{"metrics", "commands", "update"}),
+			expectedFields: []string{"arrayFilters", "failed", "pipeline", "total"},
+			expectedNoZero: []string{"total"},
+		},
+		"UpdateCmdFailed": {
+			cmds: []bson.D{
+				{{"update", int32(1)}},
+			},
+			metricsPath:    types.NewPath([]string{"metrics", "commands", "update"}),
+			expectedFields: []string{"arrayFilters", "failed", "pipeline", "total"},
+			expectedNoZero: []string{"failed", "total"},
+		},
+		// TODO: https://github.com/FerretDB/FerretDB/issues/9
+	} {
+		name, tc := name, tc
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, collection := setup.Setup(t)
+
+			for _, cmd := range tc.cmds {
+				collection.Database().RunCommand(ctx, cmd)
+			}
+
+			command := bson.D{{"serverStatus", int32(1)}}
+
+			var actual bson.D
+			err := collection.Database().RunCommand(ctx, command).Decode(&actual)
+			require.NoError(t, err)
+
+			actualMetric, err := ConvertDocument(t, actual).GetByPath(tc.metricsPath)
+			assert.NoError(t, err)
+
+			actualDoc, ok := actualMetric.(*types.Document)
+			require.True(t, ok)
+
+			actualFields := actualDoc.Keys()
+
+			sort.Strings(tc.expectedFields)
+			sort.Strings(actualFields)
+
+			assert.Equal(t, tc.expectedFields, actualFields)
+
+			var actualNotZeros []string
+			for key, value := range actualDoc.Map() {
+				assert.IsType(t, int64(0), value)
+
+				if value != 0 {
+					actualNotZeros = append(actualNotZeros, key)
+				}
+			}
+
+			for _, expectedName := range tc.expectedNoZero {
+				assert.Contains(t, actualNotZeros, expectedName)
+			}
+		})
+	}
+}
+
 // TestCommandsAdministrationWhatsMyURI tests the `whatsmyuri` command.
 // It connects two clients to the same server and checks that `whatsmyuri` returns different ports for these clients.
 func TestCommandsAdministrationWhatsMyURI(t *testing.T) {

integration/setup/common.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -30,8 +31,11 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 
 	"github.com/FerretDB/FerretDB/internal/clientconn"
+	"github.com/FerretDB/FerretDB/internal/clientconn/connmetrics"
+	"github.com/FerretDB/FerretDB/internal/handlers/common"
 	"github.com/FerretDB/FerretDB/internal/handlers/registry"
 	"github.com/FerretDB/FerretDB/internal/util/debug"
 	"github.com/FerretDB/FerretDB/internal/util/logging"
@@ -91,9 +95,15 @@ func SkipForPostgresWithReason(tb testing.TB, reason string) {
 func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) int {
 	tb.Helper()
 
+	cmdsList := maps.Keys(common.Commands)
+	sort.Strings(cmdsList)
+
+	metrics := connmetrics.NewListenerMetrics(cmdsList)
+
 	h, err := registry.NewHandler(*handlerF, &registry.NewHandlerOpts{
 		Ctx:           ctx,
 		Logger:        logger,
+		Metrics:       metrics.ConnMetrics,
 		PostgreSQLURL: testutil.PostgreSQLURL(tb, nil),
 		TigrisURL:     testutil.TigrisURL(tb),
 	})
@@ -109,6 +119,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) int {
 		ListenAddr:         "127.0.0.1:0",
 		ProxyAddr:          proxyAddr,
 		Mode:               mode,
+		Metrics:            metrics,
 		Handler:            h,
 		Logger:             logger,
 		TestRunCancelDelay: time.Hour, // make it easier to notice missing client's disconnects