-
Notifications
You must be signed in to change notification settings - Fork 109
fix: The save function should not throw an error for name/value properties in arrays #1336
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
b337a5c
59889b5
3ca8d1c
d2c0138
3307e7e
03bd79e
63427a6
7c8b7be
111a968
2587728
bc963cc
ce23c71
c96cf51
3aef723
83cfff5
b1a4dcf
891bc65
459a3e0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -991,16 +991,6 @@ export namespace entity { | |
| const MAX_DATASTORE_VALUE_LENGTH = 1500; | ||
| if (Array.isArray(entities)) { | ||
| for (const entry of entities) { | ||
| if (entry && entry.name && entry.value) { | ||
| if ( | ||
| is.string(entry.value) && | ||
| Buffer.from(entry.value).length > MAX_DATASTORE_VALUE_LENGTH | ||
| ) { | ||
| entry.excludeFromIndexes = true; | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this assignment in will apply |
||
| } else { | ||
| continue; | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We don't want this |
||
| } | ||
| } | ||
| findLargeProperties_(entry, path.concat('[]'), properties); | ||
| } | ||
| } else if (is.object(entities)) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -348,6 +348,37 @@ async.each( | |
| await datastore.delete(postKey); | ||
| }); | ||
|
|
||
| it('should save a nested name/value pair with name as a long string', async () => { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test contains the code that will throw the error without the source code changes from this PR. |
||
| const longString = Buffer.alloc(1501, '.').toString(); | ||
| const postKey = datastore.key(['Post', 'post1']); | ||
| await datastore.save({ | ||
| key: postKey, | ||
| data: { | ||
| metadata: [ | ||
| { | ||
| name: longString, | ||
| value: 'some-value', | ||
| }, | ||
| ], | ||
| }, | ||
| excludeLargeProperties: true, | ||
| }); | ||
| }); | ||
| it('should save a nested name/value pair with value as a long string', async () => { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test shows that even though we are omitting the code that appends |
||
| const longString = Buffer.alloc(1501, '.').toString(); | ||
| const postKey = datastore.key(['Post', 'post1']); | ||
| await datastore.save({ | ||
| key: postKey, | ||
| data: [ | ||
| { | ||
| name: 'some-name', | ||
| value: longString, | ||
| }, | ||
| ], | ||
| excludeLargeProperties: true, | ||
| }); | ||
| }); | ||
|
|
||
| describe('multi-db support for read and write operations', () => { | ||
| const namespace = `${Date.now()}`; | ||
| const keyHierarchy = ['Post', 'post1']; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| // Copyright 2024 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| import * as assert from 'assert'; | ||
| import {describe} from 'mocha'; | ||
| import {Entities, entity} from '../../src/entity'; | ||
| import findLargeProperties_ = entity.findLargeProperties_; | ||
|
|
||
| const async = require('async'); | ||
|
|
||
| describe('findLargeProperties_', () => { | ||
| const longString = Buffer.alloc(1501, '.').toString(); | ||
| // complexCaseEntities are passed into save for the complex case. | ||
| const complexCaseEntities = { | ||
| longString, | ||
| notMetadata: true, | ||
| longStringArray: [longString], | ||
| metadata: { | ||
| longString, | ||
| otherProperty: 'value', | ||
| obj: { | ||
| longStringArray: [ | ||
| { | ||
| longString, | ||
| nestedLongStringArray: [ | ||
| { | ||
| longString, | ||
| nestedProperty: true, | ||
| }, | ||
| { | ||
| longString, | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| }, | ||
| longStringArray: [ | ||
| { | ||
| longString, | ||
| nestedLongStringArray: [ | ||
| { | ||
| longString, | ||
| nestedProperty: true, | ||
| }, | ||
| { | ||
| longString, | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| }, | ||
| }; | ||
| async.each( | ||
| [ | ||
| { | ||
| name: 'For a simple case involving a name/value pair', | ||
| entities: { | ||
| name: 'firstElementName', | ||
| value: longString, | ||
| }, | ||
| skipped: false, | ||
| expectedOutput: ['value'], | ||
| }, | ||
| { | ||
| name: 'For a simple case involving a name/value pair in an array', | ||
| entities: [ | ||
| { | ||
| name: 'firstElementName', | ||
| value: longString, | ||
| }, | ||
| ], | ||
| skipped: false, | ||
| expectedOutput: ['[].value'], | ||
| }, | ||
| { | ||
| name: 'For a complex case involving lots of entities', | ||
| entities: complexCaseEntities, | ||
| skipped: false, | ||
| expectedOutput: [ | ||
| 'longString', | ||
| 'longStringArray[]', | ||
| 'metadata.longString', | ||
| 'metadata.obj.longStringArray[].longString', | ||
| 'metadata.obj.longStringArray[].nestedLongStringArray[].longString', | ||
| 'metadata.longStringArray[].longString', | ||
| 'metadata.longStringArray[].nestedLongStringArray[].longString', | ||
| ], | ||
| }, | ||
| { | ||
| name: 'For a complex case involving a name/value pair', | ||
| entities: { | ||
| name: 'firstElementName', | ||
| value: complexCaseEntities, | ||
| }, | ||
| skipped: false, | ||
| expectedOutput: [ | ||
| 'value.longString', | ||
| 'value.longStringArray[]', | ||
| 'value.metadata.longString', | ||
| 'value.metadata.obj.longStringArray[].longString', | ||
| 'value.metadata.obj.longStringArray[].nestedLongStringArray[].longString', | ||
| 'value.metadata.longStringArray[].longString', | ||
| 'value.metadata.longStringArray[].nestedLongStringArray[].longString', | ||
| ], | ||
| }, | ||
| { | ||
| name: 'For a complex case involving and array and name/value', | ||
| entities: [ | ||
| { | ||
| name: 'firstElementName', | ||
| value: complexCaseEntities, | ||
| }, | ||
| ], | ||
| skipped: false, | ||
| expectedOutput: [ | ||
| '[].value.longString', | ||
| '[].value.longStringArray[]', | ||
| '[].value.metadata.longString', | ||
| '[].value.metadata.obj.longStringArray[].longString', | ||
| '[].value.metadata.obj.longStringArray[].nestedLongStringArray[].longString', | ||
| '[].value.metadata.longStringArray[].longString', | ||
| '[].value.metadata.longStringArray[].nestedLongStringArray[].longString', | ||
| ], | ||
| }, | ||
| { | ||
| name: 'For some nested properties that happen to be called value and name', | ||
| entities: { | ||
| metadata: [ | ||
| { | ||
| name: longString, | ||
| value: 'some-value', | ||
| }, | ||
| ], | ||
| }, | ||
| skipped: false, | ||
| expectedOutput: ['metadata[].name'], | ||
| }, | ||
| ], | ||
| (test: { | ||
| name: string; | ||
| entities: Entities; | ||
| expectedOutput: string[]; | ||
| skipped: boolean; | ||
| }) => { | ||
| it(test.name, function () { | ||
| if (test.skipped) { | ||
| this.skip(); | ||
| } | ||
| const output = findLargeProperties_(test.entities, '', []); | ||
| assert.deepStrictEqual(output, test.expectedOutput); | ||
| }); | ||
| } | ||
| ); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -263,6 +263,31 @@ describe('Commit', () => { | |
| }, | ||
| ], | ||
| }, | ||
| { | ||
| name: 'should pass the right properties for a name/value pair in an array with a long string', | ||
| skipped: false, | ||
| entities: [ | ||
| { | ||
| name: 'entityName', | ||
| value: longString, | ||
| }, | ||
| ], | ||
| excludeFromIndexes: [], // Empty because excludeLargeProperties populates the list. | ||
| excludeLargeProperties: true, | ||
| expectedMutations: [ | ||
| { | ||
| upsert: { | ||
| properties: { | ||
| entityName: { | ||
| stringValue: longString, | ||
| excludeFromIndexes: true, | ||
| }, | ||
| }, | ||
| key, | ||
| }, | ||
| }, | ||
| ], | ||
| }, | ||
| { | ||
| name: 'should position excludeFromIndexes in the right place when provided at the top level', | ||
| skipped: false, | ||
|
|
@@ -479,6 +504,48 @@ describe('Commit', () => { | |
| }, | ||
| ], | ||
| }, | ||
| { | ||
| name: 'should set the right properties for a nested name/value pair', | ||
| skipped: false, | ||
| entities: { | ||
| metadata: [ | ||
| { | ||
| name: longString, | ||
| value: 'some-value', | ||
| }, | ||
| ], | ||
| }, | ||
| excludeFromIndexes: [], | ||
| excludeLargeProperties: true, | ||
| expectedMutations: [ | ||
| { | ||
| upsert: { | ||
| key, | ||
| properties: { | ||
| metadata: { | ||
| arrayValue: { | ||
| values: [ | ||
| { | ||
| entityValue: { | ||
| properties: { | ||
| name: { | ||
| stringValue: longString, | ||
| excludeFromIndexes: true, | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The source code changes from this PR make sure that |
||
| }, | ||
| value: { | ||
| stringValue: 'some-value', | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| ], | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| (test: { | ||
| name: string; | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The original intent of this
ifstatement was to deal with arrays passed into the save function. When arrays are passed into the save function, they are normally expected to havenameandvalueproperties defined at the top level and then their proto is encoded using the name as the key so this code addsexcludeFromIndexes:truein the right places for this special encoding.The problem with this if statement is that this function is recursive so this logic will be applied recursively and
excludeFromIndexes: truewill be added recursively, but we only want this logic to be applied at the top level for arrays because the special encoding is only done for arrays at the top level. As it turns out excludeFromIndexes is applied in the right places of the top level here instead so this if block is no longer required and a new testshould save a nested name/value pair with value as a long stringproves this.