Skip to content

Commit

Permalink
Release (#411)
Browse files Browse the repository at this point in the history 
* refac: refactor transaction handling to optimize writes

* feat: added minimal pubsub implementation

* feat: updated minimal pubsub implementation per feedback

* feat: Added support for oauth

* chore: Fix GetInfo test. Fix unneeded quota manager assignment

* refactor: Improve integrations tests

* Allow to run/debug individual tests in IDE
* Fix local_run target. Now FDB on the host should not be needed.
* Fix exit code of `make test`

* fix: Fix search response marshalling

* chore: Added auth failure logs

* feat: db and collection name tags in traces (#402)

* feat: Facet stats support for numeric fields (#409)

* feat: Facet stats support for numeric fields

* chore: Updating gh action env ubuntu to 22.04

* test: Fix marshaler test to consume pointer

* chore: Installing proto compiler from gh releases

Co-authored-by: Himank Chaudhary <himank@tigrisdata.com>
Co-authored-by: Matt Ayres <matt.ayres@gmail.com>
Co-authored-by: Jigar Joshi <jigarjm@gmail.com>
Co-authored-by: Yevgeniy Firsov <firsov@tigrisdata.com>
Co-authored-by: Peter Boros <pboros@tigrisdata.com>
  • Loading branch information
@adilansari @himank
@mattayres @JigarJoshi @efirs @pboros
6 people committed Aug 10, 2022
1 parent 468ef86 commit 9411d91
Show file tree
Hide file tree
Showing 58 changed files with 1,681 additions and 1,049 deletions.
5 changes: 3 additions & 2 deletions Makefile
View file
Expand Up @@ -50,6 +50,7 @@ docker_compose_build:
# current user owner instead of root
docker_test: generate
$(DOCKER_COMPOSE) up --build tigris_test tigris_test
@[ $$(docker inspect tigris_test --format='{{.State.ExitCode}}') = "0" ]

docker_test_no_build:
$(DOCKER_COMPOSE) up --exit-code-from tigris_test --no-build tigris_test tigris_test
Expand All @@ -64,9 +65,9 @@ run: generate
$(DOCKER_COMPOSE) up --build --detach tigris_server2

local_run: server
$(DOCKER_COMPOSE) up --no-build --detach tigris_search tigris_db
$(DOCKER_COMPOSE) up --no-build --detach tigris_search tigris_db2
fdbcli -C ./test/config/fdb.cluster --exec "configure new single memory" || true
TIGRIS_ENVIRONMENT=dev ./server/service
./server/service -c config/server.dev.yaml

# Runs tigris server and foundationdb, plus additional tools for it like:
# - prometheus and grafana for monitoring
Expand Down
2 changes: 1 addition & 1 deletion api/proto
Submodule proto updated from bcf939 to 21374e
9 changes: 9 additions & 0 deletions api/server/v1/api.go
View file
Expand Up @@ -27,3 +27,12 @@ type Request interface {
type Response interface {
proto.Message
}

type RequestWithDb interface {
GetDb() string
}

type RequestWithDbAndCollection interface {
GetDb() string
GetCollection() string
}
29 changes: 21 additions & 8 deletions api/server/v1/marshaler.go
View file
Expand Up @@ -85,7 +85,7 @@ func (x *ReadRequest) UnmarshalJSON(data []byte) error {
return nil
}

// UnmarshalJSON for SearchRequest
// UnmarshalJSON for SearchRequest avoids unmarshalling filter, facets, sort and fields
func (x *SearchRequest) UnmarshalJSON(data []byte) error {
var mp map[string]jsoniter.RawMessage
if err := jsoniter.Unmarshal(data, &mp); err != nil {
Expand Down Expand Up @@ -302,6 +302,17 @@ func (x *CreateOrUpdateCollectionRequest) UnmarshalJSON(data []byte) error {
if err := jsoniter.Unmarshal(value, &x.Options); err != nil {
return err
}
case "type":
var t string
if err := jsoniter.Unmarshal(value, &t); err != nil {
return err
}
switch t {
case "documents":
x.Type = CollectionType_DOCUMENTS
case "messages":
x.Type = CollectionType_MESSAGES
}
}
}
return nil
Expand Down Expand Up @@ -427,6 +438,9 @@ func (x *ReadResponse) MarshalJSON() ([]byte, error) {
return json.Marshal(resp)
}

// Explicit custom marshalling of some search data structures required
// to retain schema in the output even when fields are empty.

func (x *SearchResponse) MarshalJSON() ([]byte, error) {
resp := struct {
Hits []*SearchHit `json:"hits"`
Expand Down Expand Up @@ -461,7 +475,7 @@ func (x *SearchHit) MarshalJSON() ([]byte, error) {
func (x *SearchMetadata) MarshalJSON() ([]byte, error) {
resp := struct {
Found int64 `json:"found"`
TotalPages int32 `json:"totalPages"`
TotalPages int32 `json:"total_pages"`
Page *Page `json:"page"`
}{
Found: x.Found,
Expand All @@ -487,11 +501,11 @@ func (x *SearchFacet) MarshalJSON() ([]byte, error) {

func (x *FacetStats) MarshalJSON() ([]byte, error) {
resp := struct {
Avg float64 `json:"avg,omitempty"`
Max float64 `json:"max,omitempty"`
Min float64 `json:"min,omitempty"`
Sum float64 `json:"sum,omitempty"`
Count int64 `json:"count"`
Avg *float64 `json:"avg,omitempty"`
Max *float64 `json:"max,omitempty"`
Min *float64 `json:"min,omitempty"`
Sum *float64 `json:"sum,omitempty"`
Count int64 `json:"count"`
}{
Avg: x.Avg,
Max: x.Max,
Expand All @@ -505,7 +519,6 @@ func (x *FacetStats) MarshalJSON() ([]byte, error) {
type SearchHitMetadata struct {
CreatedAt *time.Time `json:"created_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
DeletedAt *time.Time `json:"deleted_at,omitempty"`
}

type Metadata struct {
Expand Down
8 changes: 5 additions & 3 deletions api/server/v1/marshaler_test.go
View file
Expand Up @@ -47,6 +47,7 @@ func TestJSONEncoding(t *testing.T) {
})

t.Run("marshal SearchResponse", func(t *testing.T) {
avg := float64(40)
resp := &SearchResponse{
Hits: []*SearchHit{{
Data: nil,
Expand All @@ -60,19 +61,20 @@ func TestJSONEncoding(t *testing.T) {
}},
Stats: &FacetStats{
Count: 50,
Avg: 40,
Avg: &avg,
},
},
},
Meta: &SearchMetadata{
Found: 1234,
TotalPages: 0,
Found: 1234,
Page: &Page{
Current: 2,
Size: 10,
},
}}
r, err := json.Marshal(resp)
require.NoError(t, err)
require.Equal(t, []byte(`{"hits":[{"metadata":{}}],"facets":{"myField":{"counts":[{"count":32,"value":"adidas"}],"stats":{"avg":40,"count":50}}},"meta":{"found":1234,"totalPages":0,"page":{"current":2,"size":10}}}`), r)
require.JSONEq(t, `{"hits":[{"metadata":{}}],"facets":{"myField":{"counts":[{"count":32,"value":"adidas"}],"stats":{"avg":40,"count":50}}},"meta":{"found":1234,"total_pages":0,"page":{"current":2,"size":10}}}`, string(r))
})
}
3 changes: 3 additions & 0 deletions config/server.dev.yaml
View file
Expand Up @@ -16,6 +16,9 @@ server:
host: localhost
port: 8081

cdc:
enabled: true

search:
host: localhost
port: 8108
Expand Down
2 changes: 2 additions & 0 deletions docker/Dockerfile
View file
Expand Up @@ -22,6 +22,8 @@ RUN apt-get update && \
curl \
gcc \
git \
unzip \
libc6-dev \
make \
sudo

Expand Down
2 changes: 2 additions & 0 deletions docker/Dockerfile.local
View file
Expand Up @@ -21,6 +21,8 @@ RUN apt-get update && \
curl \
wget \
gcc \
libc6-dev \
unzip \
git \
make \
sudo
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Expand Up @@ -29,7 +29,7 @@ require (
github.com/spf13/viper v1.12.0
github.com/stretchr/testify v1.7.2
github.com/tigrisdata/tigris-client-go v1.0.0-alpha.21
github.com/typesense/typesense-go v0.5.0
github.com/typesense/typesense-go v0.6.0
github.com/uber-go/tally v3.5.0+incompatible
github.com/ugorji/go/codec v1.2.7
github.com/valyala/bytebufferpool v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Expand Up @@ -873,8 +873,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/twitchtv/twirp v8.1.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=
github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/typesense/typesense-go v0.5.0 h1:3uit/Ku1k0wtUDjN5HfFMIoR55pEeL7gcc+fL8c+xNI=
github.com/typesense/typesense-go v0.5.0/go.mod h1:F9T3neLDqRr9ufFNhv1y0Qxe1Zs1GT85JlgijSjtKFo=
github.com/typesense/typesense-go v0.6.0 h1:QcQ+1WGqFDyBhjP7Cd8mj7cs00zFTs6uBrB2u6we78o=
github.com/typesense/typesense-go v0.6.0/go.mod h1:F9T3neLDqRr9ufFNhv1y0Qxe1Zs1GT85JlgijSjtKFo=
github.com/uber-go/tally v3.5.0+incompatible h1:2vIkqVrSaspifqcJh2yQjQqqpfavvmfj/ognDrBxuSg=
github.com/uber-go/tally v3.5.0+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
Expand Down
23 changes: 16 additions & 7 deletions schema/schema.go
View file
Expand Up @@ -95,9 +95,19 @@ type Factory struct {

// Build is used to deserialize the user json schema into a schema factory.
func Build(collection string, reqSchema jsoniter.RawMessage) (*Factory, error) {
return BuildWithType(collection, reqSchema, api.CollectionType_DOCUMENTS)
}

func BuildWithType(collection string, reqSchema jsoniter.RawMessage, cType api.CollectionType) (*Factory, error) {
var err error
if reqSchema, err = addPrimaryKeyIfMissing(reqSchema); err != nil {
return nil, err
if cType == api.CollectionType_MESSAGES {
if reqSchema, err = setPrimaryKey(reqSchema, jsonSpecFormatDateTime, false); err != nil {
return nil, err
}
} else {
if reqSchema, err = setPrimaryKey(reqSchema, jsonSpecFormatUUID, true); err != nil {
return nil, err
}
}

var schema = &JSONSchema{}
Expand Down Expand Up @@ -149,13 +159,13 @@ func Build(collection string, reqSchema jsoniter.RawMessage) (*Factory, error) {
}, nil
}

func addPrimaryKeyIfMissing(reqSchema jsoniter.RawMessage) (jsoniter.RawMessage, error) {
func setPrimaryKey(reqSchema jsoniter.RawMessage, format string, ifMissing bool) (jsoniter.RawMessage, error) {
var schema map[string]interface{}
if err := jsoniter.Unmarshal(reqSchema, &schema); err != nil {
return nil, err
}

if _, ok := schema[PrimaryKeySchemaK]; ok {
if _, ok := schema[PrimaryKeySchemaK]; ifMissing && ok {
// primary key exists, no need to do anything.
return reqSchema, nil
}
Expand All @@ -167,11 +177,10 @@ func addPrimaryKeyIfMissing(reqSchema jsoniter.RawMessage) (jsoniter.RawMessage,
return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "properties object is invalid")
}

if _, ok = propertiesMap[AutoPrimaryKeyF]; !ok {
// if user doesn't have the ID field then add it of type UUID
if _, ok = propertiesMap[AutoPrimaryKeyF]; !ifMissing || !ok {
propertiesMap[AutoPrimaryKeyF] = map[string]interface{}{
"type": jsonSpecString,
"format": jsonSpecFormatUUID,
"format": format,
"autoGenerate": true,
}
}
Expand Down
128 changes: 77 additions & 51 deletions scripts/install_build_deps.sh
View file
Expand Up @@ -16,6 +16,8 @@ set -ex

# Settings
FDB_VERSION=7.1.7
PROTO_VERSION=3.15.8
PROTO_RELEASES="https://github.com/protocolbuffers/protobuf/releases"

### Prereqs checks ###
# Check if architecture and OS is supported
Expand All @@ -24,74 +26,98 @@ ARCH=$(uname -m)
OS=$(uname -s)

case "${OS}-${ARCH}" in
"Darwin-arm64")
BINARIES="brew curl go"
FDB_SHA=b456a1d03580f81394502e1b066006ec38bf6a3a17a9904e6d7a88badbea4b4a08b9dbf69fbb0057b46dd5043f23de9ec3ff7a77d767c23f872425eb73fdee18
;;
"Darwin-x86_64")
BINARIES="brew curl go"
FDB_SHA=0e9d147410eede58d121fdc9208ea7b04a7c74c8f3776f56cfc00233cfb3a358a0e2f992122718ef9c639928b081801da542e9c4b07a539c8fd73361ab43beec
;;
"Linux-aarch64")
BINARIES="apt-get curl go"
FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
;;
"Linux-arm64")
BINARIES="apt-get curl go"
FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
;;
"Linux-x86_64")
BINARIES="apt-get curl go"
FDB_SHA=471f6bf4a7af40abc69027aa0d4f452ee83715a43a555008303ca255f6bd6db1
;;
*)
echo "Unsupported architecture ${ARCH} or operating system ${OS}."
exit 1
"Darwin-arm64")
BINARIES="brew curl go"
FDB_SHA=b456a1d03580f81394502e1b066006ec38bf6a3a17a9904e6d7a88badbea4b4a08b9dbf69fbb0057b46dd5043f23de9ec3ff7a77d767c23f872425eb73fdee18
;;
"Darwin-x86_64")
BINARIES="brew curl go"
FDB_SHA=0e9d147410eede58d121fdc9208ea7b04a7c74c8f3776f56cfc00233cfb3a358a0e2f992122718ef9c639928b081801da542e9c4b07a539c8fd73361ab43beec
;;
"Linux-aarch64")
BINARIES="apt-get curl go"
FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
;;
"Linux-arm64")
BINARIES="apt-get curl go"
FDB_SHA=c994ebb01a660cff9ef699a0e38482a561679db04c02c256efae25ba687cf903
;;
"Linux-x86_64")
BINARIES="apt-get curl go"
FDB_SHA=471f6bf4a7af40abc69027aa0d4f452ee83715a43a555008303ca255f6bd6db1
;;
*)
echo "Unsupported architecture ${ARCH} or operating system ${OS}."
exit 1
;;
esac

# Check if required binaries are available in PATH
for bin in ${BINARIES}; do
binpath=$(command -v "${bin}")
if [ -z "${binpath}" ] || ! test -x "${binpath}"; then
echo "Please ensure that $bin binary is installed and in PATH."
exit 1
fi
binpath=$(command -v "${bin}")
if [ -z "${binpath}" ] || ! test -x "${binpath}"; then
echo "Please ensure that $bin binary is installed and in PATH."
exit 1
fi
done

# Install protobuf compiler via package manager
# Install protobuf compiler
case "${OS}" in
"Darwin")
brew install protobuf
;;
"Linux")
sudo apt-get update
sudo apt-get install -y protobuf-compiler
;;
"Darwin")
brew install protobuf
;;
"Linux")
case "${ARCH}" in
"x86_64")
PROTO_PKG=protoc-$PROTO_VERSION-linux-x86_64.zip
;;
"aarch64")
PROTO_PKG=protoc-$PROTO_VERSION-linux-aarch_64.zip
;;
*)
echo "No supported proto compiler for ${ARCH} or operating system ${OS}."
exit 1
;;
esac
;;
*)
echo "No supported proto compiler for ${ARCH} or operating system ${OS}."
exit 1
;;
esac

if [ -n "$PROTO_PKG" ]; then
DOWNLOAD_URL=$PROTO_RELEASES/download/v$PROTO_VERSION/$PROTO_PKG
echo "Fetching protobuf release ${DOWNLOAD_URL}"
curl -LO $DOWNLOAD_URL
sudo unzip $PROTO_PKG -d "/usr/local/"
sudo chmod +x "/usr/local/bin/protoc"
sudo chmod -R 755 "/usr/local/include/"
fi

# Install protobuf
export GO111MODULE=on
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1
go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2
go install github.com/google/gnostic/cmd/protoc-gen-openapi@v0 #generate openapi 3.0 spec
go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1 #generate go http client
go install github.com/mikefarah/yq/v4@latest # used to fix OpenAPI spec in scripts/fix_openapi.sh
go install github.com/mikefarah/yq/v4@latest # used to fix OpenAPI spec in scripts/fix_openapi.sh

# Install FoundationDB package
case "${OS}" in
"Darwin")
FDB_PACKAGE_NAME="FoundationDB-${FDB_VERSION}_${ARCH}.pkg"
FDB_PACKAGE_PATH="$(mktemp -d)/${FDB_PACKAGE_NAME}"
curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/${FDB_PACKAGE_NAME}"
echo "$FDB_SHA $FDB_PACKAGE_PATH" | shasum -a 512 -c
sudo installer -pkg "$FDB_PACKAGE_PATH" -target /
;;
"Linux")
FDB_PACKAGE_NAME="foundationdb-clients_${FDB_VERSION}-1_${ARCH}.deb"
FDB_PACKAGE_PATH="$(mktemp -p /tmp/ -u)/${FDB_PACKAGE_NAME}"
curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/ubuntu/focal/${FDB_PACKAGE_NAME}"
echo "$FDB_SHA $FDB_PACKAGE_PATH" | sha256sum -c
sudo dpkg -i "$FDB_PACKAGE_PATH" # provides /lib/libfdb_c.so shared library in the docker for CGO
;;
"Darwin")
FDB_PACKAGE_NAME="FoundationDB-${FDB_VERSION}_${ARCH}.pkg"
FDB_PACKAGE_PATH="$(mktemp -d)/${FDB_PACKAGE_NAME}"
curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/${FDB_PACKAGE_NAME}"
echo "$FDB_SHA $FDB_PACKAGE_PATH" | shasum -a 512 -c
sudo installer -pkg "$FDB_PACKAGE_PATH" -target /
;;
"Linux")
FDB_PACKAGE_NAME="foundationdb-clients_${FDB_VERSION}-1_${ARCH}.deb"
FDB_PACKAGE_PATH="$(mktemp -p /tmp/ -u)/${FDB_PACKAGE_NAME}"
curl --create-dirs -Lo "$FDB_PACKAGE_PATH" "https://tigrisdata-pub.s3.us-west-2.amazonaws.com/ubuntu/focal/${FDB_PACKAGE_NAME}"
echo "$FDB_SHA $FDB_PACKAGE_PATH" | sha256sum -c
sudo dpkg -i "$FDB_PACKAGE_PATH" # provides /lib/libfdb_c.so shared library in the docker for CGO
;;
esac

0 comments on commit 9411d91

Please sign in to comment.