Update documentation about fjson package (#262)

Closes #182.

.golangci.yml

@@ -6,6 +6,10 @@ run:
   issues-exit-code: 0
 
 linters-settings:
+  depguard:
+    list-type: blacklist
+    packages:
+      - github.com/FerretDB/FerretDB/internal/bson
   exhaustive:
     default-signifies-exhaustive: false
   goconst:
@@ -120,3 +124,8 @@ linters:
 
 issues:
   exclude-use-default: false
+  exclude-rules:
+    # only `wire` package can import `bson` package
+    - linters: [depguard]
+      path: internal/wire
+      text: bson

CONTRIBUTING.md

@@ -41,3 +41,22 @@ $ git remote add origin git@github.com:<YOUR_GITHUB_USERNAME>/FerretDB.git
    This allows you to run commands against FerretDB.
 
 You can see all available "make" commands with `make help`.
+
+## Code Overview
+
+Package `cmd` provides commands implementation. `ferretdb` is the main FerretDB binary; others are tools for development.
+
+Package `tools` uses "tools.go" approach to fix tools versions. They are installed into `bin/` by `make init`.
+
+`internal` subpackages contain most of the FerretDB code:
+* `types` package provides Go types matching BSON types that don't have built-in Go equivalents: we use `int32` for BSON's int32, but types.ObjectID for BSON's ObjectId.
+* `fjson` provides converters from/to FJSON for built-in and `types` types.
+  FJSON adds some extensions to JSON for keeping object keys in order, preserving BSON type information, etc.
+  FJSON is used by `jsonb1` handler/storage.
+* `bson` package provides converters from/to BSON for built-in and `types` types.
+* `wire` package provides wire protocol implementation.
+* `clientconn` package provides client connection implementation.
+  It accepts client connections, reads `wire`/`bson` protocol messages, and passes them to `handlers`.
+  Responses are then converted to `wire`/`bson` messages and sent back to the client.
+* `handlers` handle protocol commands.
+  They use `fjson` package for storing data in PostgreSQL in jsonb columns, but they don't use `bson` package – all data is represented as built-in and `types` types.

internal/bson/bson.go

@@ -12,15 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package bson provides converters from/to BSON.
+// Package bson provides converters from/to BSON for built-in and `types` types.
 //
-// All BSON data types have three representations in FerretDB:
-//
-//  1. As they are used in handlers implementation (types package).
-//  2. As they are used in the wire protocol implementation (bson package).
-//  3. As they are used to store data in PostgreSQL (fjson package).
-//
-// The reason for that is a separation of concerns: to avoid method names clashes, to simplify type asserts, etc.
+// See contributing guidelines and documentation for package `types` for details.
 package bson
 
 import (

internal/bson/document.go

@@ -27,7 +27,8 @@ import (
 )
 
 const (
-	MaxDocumentLen = 16777216
+	// Deprecated: use types.MaxDocumentLen instead.
+	MaxDocumentLen = types.MaxDocumentLen
 
 	minDocumentLen = 5
 )

internal/clientconn/conn.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package clientconn provides client connection implementation.
 package clientconn
 
 import (

internal/fjson/fjson.go

@@ -12,35 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package fjson provides converters from/to FJSON.
+// Package fjson provides converters from/to FJSON (JSON with some extensions) for built-in and `types` types.
 //
-// All BSON data types have three representations in FerretDB:
+// See contributing guidelines and documentation for package `types` for details.
 //
-//  1. As they are used in handlers implementation (types package).
-//  2. As they are used in the wire protocol implementation (bson package).
-//  3. As they are used to store data in PostgreSQL (fjson package).
+// Mapping
 //
-// The reason for that is a separation of concerns: to avoid method names clashes, to simplify type asserts, etc.
-//
-// JSON mapping for storage
-//
-// Composite/pointer types
-//  Document:   {"$k": ["<key 1>", "<key 2>", ...], "<key 1>": <value 1>, "<key 2>": <value 2>, ...}
-//  Array:      JSON array
-// Scalar/value types
-//  Double:     {"$f": JSON number} or {"$f": "Infinity|-Infinity|NaN"}
-//  String:     JSON string
-//  Binary:     {"$b": "<base 64 string>", "s": <subtype number>}
-//  ObjectID:   {"$o": "<ObjectID as 24 character hex string"}
-//  Bool:       JSON true / false values
-//  DateTime:   {"$d": milliseconds since epoch as JSON number}
-//  nil:        JSON null
-//  Regex:      {"$r": "<string without terminating 0x0>", "o": "<string without terminating 0x0>"}
-//  Int32:      JSON number
-//  Timestamp:  {"$t": "<number as string>"}
-//  Int64:      {"$l": "<number as string>"}
-//  Decimal128: {"$n": "<number as string>"}
-//  CString:    {"$c": "<string without terminating 0x0>"}
+// Composite types
+//  types.Document   {"$k": ["<key 1>", "<key 2>", ...], "<key 1>": <value 1>, "<key 2>": <value 2>, ...}
+//  *types.Array     JSON array
+// Scalar types
+//  float64          {"$f": JSON number} or {"$f": "Infinity|-Infinity|NaN"}
+//  string           JSON string
+//  types.Binary     {"$b": "<base 64 string>", "s": <subtype number>}
+//  types.ObjectID   {"$o": "<ObjectID as 24 character hex string"}
+//  bool             JSON true / false values
+//  time.Time        {"$d": milliseconds since epoch as JSON number}
+//  nil              JSON null
+//  types.Regex      {"$r": "<string without terminating 0x0>", "o": "<string without terminating 0x0>"}
+//  int32            JSON number
+//  types.Timestamp  {"$t": "<number as string>"}
+//  int64            {"$l": "<number as string>"}
+//  TODO Decimal128  {"$n": "<number as string>"}
+//  types.CString    {"$c": "<string without terminating 0x0>"}
 package fjson
 
 import (

internal/handlers/handler_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 
-	"github.com/FerretDB/FerretDB/internal/bson"
 	"github.com/FerretDB/FerretDB/internal/handlers/jsonb1"
 	"github.com/FerretDB/FerretDB/internal/handlers/sql"
 	"github.com/FerretDB/FerretDB/internal/pg"
@@ -433,7 +432,7 @@ func TestReadOnlyHandlers(t *testing.T) {
 				"versionArray", types.MustNewArray(int32(5), int32(0), int32(42), int32(0)),
 				"bits", int32(strconv.IntSize),
 				"debug", version.Get().Debug,
-				"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+				"maxBsonObjectSize", int32(16777216),
 				"ok", float64(1),
 				"buildEnvironment", types.MustMakeDocument(),
 			),
@@ -687,7 +686,7 @@ func TestReadOnlyHandlers(t *testing.T) {
 			resp: types.MustMakeDocument(
 				"helloOk", true,
 				"ismaster", true,
-				"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+				"maxBsonObjectSize", int32(16777216),
 				"maxMessageSizeBytes", int32(wire.MaxMsgLen),
 				"maxWriteBatchSize", int32(100000),
 				"localTime", time.Now(),
@@ -708,7 +707,7 @@ func TestReadOnlyHandlers(t *testing.T) {
 			resp: types.MustMakeDocument(
 				"helloOk", true,
 				"ismaster", true,
-				"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+				"maxBsonObjectSize", int32(16777216),
 				"maxMessageSizeBytes", int32(wire.MaxMsgLen),
 				"maxWriteBatchSize", int32(100000),
 				"localTime", time.Now(),

internal/handlers/msg_buildinfo.go

@@ -18,9 +18,9 @@ import (
 	"context"
 	"strconv"
 
-	"github.com/FerretDB/FerretDB/internal/bson"
 	"github.com/FerretDB/FerretDB/internal/types"
 	"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
+	"github.com/FerretDB/FerretDB/internal/util/must"
 	"github.com/FerretDB/FerretDB/internal/util/version"
 	"github.com/FerretDB/FerretDB/internal/wire"
 )
@@ -35,10 +35,10 @@ func (h *Handler) MsgBuildInfo(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs
 		Documents: []types.Document{types.MustMakeDocument(
 			"version", versionValue,
 			"gitVersion", version.Get().Commit,
-			"versionArray", types.MustNewArray(int32(5), int32(0), int32(42), int32(0)),
+			"versionArray", must.NotFail(types.NewArray(int32(5), int32(0), int32(42), int32(0))),
 			"bits", int32(strconv.IntSize),
 			"debug", version.Get().Debug,
-			"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+			"maxBsonObjectSize", int32(types.MaxDocumentLen),
 			"ok", float64(1),
 			"buildEnvironment", version.Get().BuildEnvironment,
 		)},

internal/handlers/msg_getcmdlineopts.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/FerretDB/FerretDB/internal/types"
 	"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
+	"github.com/FerretDB/FerretDB/internal/util/must"
 	"github.com/FerretDB/FerretDB/internal/wire"
 )
 
@@ -27,7 +28,7 @@ func (h *Handler) MsgGetCmdLineOpts(ctx context.Context, msg *wire.OpMsg) (*wire
 	var reply wire.OpMsg
 	err := reply.SetSections(wire.OpMsgSection{
 		Documents: []types.Document{types.MustMakeDocument(
-			"argv", types.MustNewArray("ferretdb"),
+			"argv", must.NotFail(types.NewArray("ferretdb")),
 			"parsed", types.MustMakeDocument(),
 			"ok", float64(1),
 		)},

internal/handlers/msg_hello.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"time"
 
-	"github.com/FerretDB/FerretDB/internal/bson"
 	"github.com/FerretDB/FerretDB/internal/types"
 	"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
 	"github.com/FerretDB/FerretDB/internal/wire"
@@ -33,7 +32,7 @@ func (h *Handler) MsgHello(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, e
 			"helloOk", true,
 			"ismaster", true,
 			// topologyVersion
-			"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+			"maxBsonObjectSize", int32(types.MaxDocumentLen),
 			"maxMessageSizeBytes", int32(wire.MaxMsgLen),
 			"maxWriteBatchSize", int32(100000),
 			"localTime", time.Now(),

internal/handlers/query_cmd.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"time"
 
-	"github.com/FerretDB/FerretDB/internal/bson"
 	"github.com/FerretDB/FerretDB/internal/handlers/common"
 	"github.com/FerretDB/FerretDB/internal/types"
 	"github.com/FerretDB/FerretDB/internal/wire"
@@ -35,7 +34,7 @@ func (h *Handler) QueryCmd(ctx context.Context, query *wire.OpQuery) (*wire.OpRe
 					"helloOk", true,
 					"ismaster", true,
 					// topologyVersion
-					"maxBsonObjectSize", int32(bson.MaxDocumentLen),
+					"maxBsonObjectSize", int32(types.MaxDocumentLen),
 					"maxMessageSizeBytes", int32(wire.MaxMsgLen),
 					"maxWriteBatchSize", int32(100000),
 					"localTime", time.Now(),

internal/types/array.go

@@ -43,6 +43,8 @@ func NewArray(values ...any) (*Array, error) {
 }
 
 // MustNewArray is a NewArray that panics in case of error.
+//
+// Deprecated: use `must.NotFail(NewArray(...))` instead.
 func MustNewArray(values ...any) *Array {
 	a, err := NewArray(values...)
 	if err != nil {

internal/types/binary.go

@@ -16,6 +16,7 @@ package types
 
 //go:generate ../../bin/stringer -linecomment -type BinarySubtype
 
+// BinarySubtype represents BSON Binary's subtype.
 type BinarySubtype byte
 
 const (
@@ -29,6 +30,7 @@ const (
 	BinaryUser       = BinarySubtype(0x80) // user
 )
 
+// Binary represents BSON type Binary.
 type Binary struct {
 	Subtype BinarySubtype
 	B       []byte

internal/types/types.go

@@ -12,22 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package types provides Go types matching BSON types without built-in Go equivalents.
+// Package types provides Go types matching BSON types that don't have built-in Go equivalents.
 //
 // All BSON data types have three representations in FerretDB:
 //
-//  1. As they are used in handlers implementation (types package).
-//  2. As they are used in the wire protocol implementation (bson package).
-//  3. As they are used to store data in PostgreSQL (fjson package).
+//  1. As they are used in "business logic" / handlers - `types` package.
+//  2. As they are used in the wire protocol implementation - `bson` package.
+//  3. As they are used to store data in PostgreSQL - `fjson` package.
 //
-// The reason for that is a separation of concerns: to avoid method names clashes, to simplify type asserts, etc.
+// The reason for that is a separation of concerns: to avoid method names clashes, to simplify type asserts,
+// to make refactorings and optimizations easier, etc.
 //
 // Mapping
 //
-// Composite/pointer types
+// Composite types
 //  types.Document   *bson.Document  *fjson.Document
 //  *types.Array     *bson.Array     *fjson.Array
-// Scalar/value types
+// Scalar types
 //  float64          *bson.Double     *fjson.Double
 //  string           *bson.String     *fjson.String
 //  types.Binary     *bson.Binary     *fjson.Binary
@@ -39,7 +40,6 @@
 //  int32            *bson.Int32      *fjson.Int32
 //  types.Timestamp  *bson.Timestamp  *fjson.Timestamp
 //  int64            *bson.Int64      *fjson.Int64
-//  TODO Decimal128
 //  types.CString    *bson.CString    *fjson.CString
 package types
 
@@ -48,7 +48,9 @@ import (
 	"time"
 )
 
-// CompositeType represents composite/pointer type - Document or *Array.
+const MaxDocumentLen = 16777216
+
+// CompositeType represents composite type - Document or *Array.
 type CompositeType interface {
 	Document | *Array
 	GetByPath(path ...string) (any, error)
@@ -59,15 +61,19 @@ type CompositeType interface {
 //go-sumtype:decl CompositeType
 
 type (
+	// CString represents BSON type CString that used as document field name, etc.
 	CString string
 
+	// ObjectID represents BSON type ObjectID.
 	ObjectID [12]byte
 
+	// Regex represents BSON type Regex.
 	Regex struct {
 		Pattern string
 		Options string
 	}
 
+	// Timestamp represents BSON type Timestamp.
 	Timestamp uint64
 )
 

internal/wire/op_msg.go

@@ -293,6 +293,8 @@ func (msg *OpMsg) MarshalBinary() ([]byte, error) {
 }
 
 // String returns a string representation for logging.
+//
+// Currently, it uses FJSON, but that may change in the future.
 func (msg *OpMsg) String() string {
 	if msg == nil {
 		return "<nil>"

internal/wire/op_query.go

@@ -131,6 +131,8 @@ func (query *OpQuery) MarshalBinary() ([]byte, error) {
 }
 
 // String returns a string representation for logging.
+//
+// Currently, it uses FJSON, but that may change in the future.
 func (query *OpQuery) String() string {
 	if query == nil {
 		return "<nil>"

internal/wire/op_reply.go

@@ -123,6 +123,8 @@ func (reply *OpReply) MarshalBinary() ([]byte, error) {
 }
 
 // String returns a string representation for logging.
+//
+// Currently, it uses FJSON, but that may change in the future.
 func (reply *OpReply) String() string {
 	if reply == nil {
 		return "<nil>"

internal/wire/wire.go

@@ -0,0 +1,16 @@
+// Copyright 2021 FerretDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package wire provides wire protocol implementation.
+package wire