Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Made dataloader case insensitive #1487

Merged
merged 3 commits into from
Jan 3, 2021
Merged

Made dataloader case insensitive #1487

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34aea81
Following changes were done
sharadregoti Jan 2, 2021
5e82c8a
Made changes according to the review
sharadregoti Jan 3, 2021
2c265a8
Merge branch 'v0.21.0' of github.com:spaceuptech/space-cloud into ce-…
sharadregoti Jan 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions gateway/model/crud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ const (
// SQLServer is the type used for MsSQL
SQLServer DBType = "sqlserver"

// DefaultValidate is used for default validation operation
DefaultValidate = "default"

// DefaultFetchLimit is the default value to be used as a limit to fetch rows/collection in each read query
DefaultFetchLimit = 1000
)
3 changes: 2 additions & 1 deletion gateway/model/graphql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ type GraphQLRequest struct {

// ReadRequestKey is the key type for the dataloader
type ReadRequestKey struct {
DBType string
DBAlias string
Col string
DBType string
HasOptions bool
Req ReadRequest
ReqParams RequestParams
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/crud/bolt/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func (b *Bolt) Delete(ctx context.Context, col string, req *model.DeleteRequest)
return helpers.Logger.LogError(helpers.GetRequestID(ctx), "Unable to unmarshal data of bbolt db", err, nil)
}
// if valid then delete
if utils.Validate(req.Find, result) {
if utils.Validate(string(model.EmbeddedDB), req.Find, result) {
// delete data
if err := bucket.Delete(k); err != nil {
return helpers.Logger.LogError(helpers.GetRequestID(ctx), "Unable to delete bbolt key", err, nil)
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/crud/bolt/read.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func (b *Bolt) Read(ctx context.Context, col string, req *model.ReadRequest) (in
if err := json.Unmarshal(v, &result); err != nil {
return helpers.Logger.LogError(helpers.GetRequestID(ctx), "Unable to unmarshal while reading from bbolt db", err, nil)
}
if utils.Validate(req.Find, result) {
if utils.Validate(string(model.EmbeddedDB), req.Find, result) {
if req.Options.Debug {
result["_dbFetchTs"] = time.Now().Format(time.RFC3339Nano)
}
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/crud/bolt/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func (b *Bolt) Update(ctx context.Context, col string, req *model.UpdateRequest)
return helpers.Logger.LogError(helpers.GetRequestID(ctx), "Unable to unmarshal data read from bbbolt db", err, nil)
}
// if valid then update
if utils.Validate(req.Find, currentObj) {
if utils.Validate(string(model.EmbeddedDB), req.Find, currentObj) {
objToSet, ok := req.Update["$set"].(map[string]interface{})
if !ok {
return helpers.Logger.LogError(helpers.GetRequestID(ctx), "Unable to update in bbolt - $set db operator not found or the operator value is not map", nil, nil)
Expand Down
11 changes: 6 additions & 5 deletions gateway/modules/crud/dataloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type resultsHolder struct {
type meta struct {
whereClause map[string]interface{}
op string
dbType string
}

type queryResult struct {
Expand Down Expand Up @@ -51,7 +52,7 @@ func (holder *resultsHolder) getWhereClauses() []interface{} {
return arr
}

func (holder *resultsHolder) addMeta(op string, whereClause map[string]interface{}, matchClause []map[string]interface{}) {
func (holder *resultsHolder) addMeta(op, dbType string, whereClause map[string]interface{}, matchClause []map[string]interface{}) {
holder.Lock()
for i, where := range matchClause {
for k, v := range where {
Expand All @@ -61,7 +62,7 @@ func (holder *resultsHolder) addMeta(op string, whereClause map[string]interface
whereClause[k] = v
}
}
holder.metas = append(holder.metas, meta{whereClause: whereClause, op: op})
holder.metas = append(holder.metas, meta{whereClause: whereClause, op: op, dbType: dbType})
holder.Unlock()
}

Expand All @@ -85,7 +86,7 @@ func (holder *resultsHolder) fillResults(metData *model.SQLMetaData, res []inter
isOperationTypeOne := meta.op == utils.One
docs := make([]interface{}, 0)
for _, doc := range res {
if utils.Validate(meta.whereClause, doc) {
if utils.Validate(meta.dbType, meta.whereClause, doc) {
docs = append(docs, doc)
}
if isOperationTypeOne {
Expand Down Expand Up @@ -160,7 +161,7 @@ func (m *Module) dataLoaderBatchFn(c context.Context, keys dataloader.Keys) []*d
for index, key := range keys {
req := key.(model.ReadRequestKey)

dbAlias = req.DBType
dbAlias = req.DBAlias
col = req.Col

// Execute query immediately if it has options
Expand Down Expand Up @@ -193,7 +194,7 @@ func (m *Module) dataLoaderBatchFn(c context.Context, keys dataloader.Keys) []*d
}

// Append the where clause to the list
holder.addMeta(req.Req.Operation, req.Req.Find, req.Req.MatchWhere)
holder.addMeta(req.Req.Operation, req.DBType, req.Req.Find, req.Req.MatchWhere)
}

// Wait for all results to be done
Expand Down
6 changes: 5 additions & 1 deletion gateway/modules/crud/operations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,11 @@ func (m *Module) Read(ctx context.Context, dbAlias, col string, req *model.ReadR
}

if req.IsBatch {
key := model.ReadRequestKey{DBType: dbAlias, Col: col, HasOptions: req.Options.HasOptions, Req: *req, ReqParams: params}
dbType, err := m.GetDBType(dbAlias)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this acquire a read lock?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dont export it if it doesn't acquire a lock

if err != nil {
return nil, nil, err
}
key := model.ReadRequestKey{DBType: dbType, DBAlias: dbAlias, Col: col, HasOptions: req.Options.HasOptions, Req: *req, ReqParams: params}
dataLoader, ok := m.getLoader(fmt.Sprintf("%s-%s-%s", m.project, dbAlias, col))
if !ok {
dataLoader = m.createLoader(fmt.Sprintf("%s-%s-%s", m.project, dbAlias, col))
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/crud/sql/integration_raw_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func TestSQL_CreateDatabaseIfNotExist(t *testing.T) {
v[colType.Name()] = string(value)
}
}
// mysqlTypeCheck(utils.DBType(*dbType), rowTypes, v)
// mysqlTypeCheck(utils.DBAlias(*dbType), rowTypes, v)
readResult = append(readResult, v)
}
if !reflect.DeepEqual(tt.want, readResult) {
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/eventing/eventing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func TestModule_SetConfig(t *testing.T) {
schemaMockArgs: []mockArgs{},
},
{
name: "DBType not mentioned",
name: "DBAlias not mentioned",
m: &Module{config: &config.Eventing{Enabled: true}},
args: args{eventing: &config.EventingConfig{Enabled: true, DBAlias: ""}},
wantErr: true,
Expand Down
2 changes: 1 addition & 1 deletion gateway/modules/realtime/routine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func (m *Module) helperSendFeed(ctx context.Context, data *model.FeedData) {
m.metrics.AddDBOperation(m.project, data.DBType, data.Group, 1, model.Read)

case utils.RealtimeInsert, utils.RealtimeUpdate:
if utils.Validate(query.whereObj, data.Payload) {
if utils.Validate(model.DefaultValidate, query.whereObj, data.Payload) {
_ = m.auth.PostProcessMethod(ctx, query.actions, dataPoint.Payload)
query.sendFeed(dataPoint)
m.metrics.AddDBOperation(m.project, data.DBType, data.Group, 1, model.Read)
Expand Down
1 change: 0 additions & 1 deletion gateway/modules/schema/creation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@ func (s *Schema) generateCreationQueries(ctx context.Context, dbAlias, tableName
continue
}
if arr := deep.Equal(fields.IndexTableProperties, cleanIndexMap(currentIndexMap[indexName].IndexTableProperties)); len(arr) > 0 {
fmt.Println("Array", arr)
batchedQueries = append(batchedQueries, s.removeIndex(dbType, dbAlias, logicalDBName, tableName, currentIndexMap[indexName].IndexName))
batchedQueries = append(batchedQueries, s.addIndex(dbType, dbAlias, logicalDBName, tableName, indexName, fields.IsIndexUnique, fields.IndexTableProperties))
}
Expand Down
Loading