From 30ec4ffe2f8308016831d6500a7b7f36ab7d31f0 Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Tue, 23 Aug 2022 11:52:56 +0200 Subject: [PATCH 1/6] feat: disable index validation --- spec/schemas.spec.js | 24 +++++++++++++++++++ .../Storage/Mongo/MongoStorageAdapter.js | 4 ++++ .../Postgres/PostgresStorageAdapter.js | 19 ++++++++++++--- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/spec/schemas.spec.js b/spec/schemas.spec.js index 9557dd7924..b1ce67232e 100644 --- a/spec/schemas.spec.js +++ b/spec/schemas.spec.js @@ -2932,6 +2932,7 @@ describe('schemas', () => { beforeEach(async () => { await TestUtils.destroyAllDataPermanently(false); await config.database.adapter.performInitialization({ VolatileClassesSchemas: [] }); + databaseAdapter.disableIndexFieldValidation = false; }); it('cannot create index if field does not exist', done => { @@ -2960,6 +2961,29 @@ describe('schemas', () => { }); }); + it('can create index if field does not exist with disableIndexFieldValidation true ', async () => { + databaseAdapter.disableIndexFieldValidation = true; + await request({ + url: 'http://localhost:8378/1/schemas/NewClass', + method: 'POST', + headers: masterKeyHeaders, + json: true, + body: {}, + }); + const response = await request({ + url: 'http://localhost:8378/1/schemas/NewClass', + method: 'PUT', + headers: masterKeyHeaders, + json: true, + body: { + indexes: { + name1: { aString: 1 }, + }, + }, + }); + expect(response.data.indexes.name1).toEqual({ aString: 1 }); + }); + it('can create index on default field', done => { request({ url: 'http://localhost:8378/1/schemas/NewClass', diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 93e23b91c3..225cb17916 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -139,6 +139,7 @@ export class MongoStorageAdapter implements StorageAdapter { _maxTimeMS: ?number; canSortOnJoinTables: boolean; enableSchemaHooks: boolean; + disableIndexFieldValidation: boolean; constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) { this._uri = uri; @@ -152,7 +153,9 @@ export class MongoStorageAdapter implements StorageAdapter { this._maxTimeMS = mongoOptions.maxTimeMS; this.canSortOnJoinTables = true; this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks; + this.disableIndexFieldValidation = !!mongoOptions.disableIndexFieldValidation; delete mongoOptions.enableSchemaHooks; + delete mongoOptions.disableIndexFieldValidation; delete mongoOptions.maxTimeMS; } @@ -287,6 +290,7 @@ export class MongoStorageAdapter implements StorageAdapter { } else { Object.keys(field).forEach(key => { if ( + !this.disableIndexFieldValidation && !Object.prototype.hasOwnProperty.call( fields, key.indexOf('_p_') === 0 ? key.replace('_p_', '') : key diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index eb668868c6..b5132d21d7 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -851,11 +851,14 @@ export class PostgresStorageAdapter implements StorageAdapter { _pgp: any; _stream: any; _uuid: any; + disableIndexFieldValidation: boolean; constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) { this._collectionPrefix = collectionPrefix; this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks; + this.disableIndexFieldValidation = !!databaseOptions.disableIndexFieldValidation; delete databaseOptions.enableSchemaHooks; + delete databaseOptions.disableIndexFieldValidation; const { client, pgp } = createClient(uri, databaseOptions); this._client = client; @@ -975,7 +978,10 @@ export class PostgresStorageAdapter implements StorageAdapter { delete existingIndexes[name]; } else { Object.keys(field).forEach(key => { - if (!Object.prototype.hasOwnProperty.call(fields, key)) { + if ( + !this.disableIndexFieldValidation && + !Object.prototype.hasOwnProperty.call(fields, key) + ) { throw new Parse.Error( Parse.Error.INVALID_QUERY, `Field ${key} does not exist, cannot add index.` @@ -990,8 +996,15 @@ export class PostgresStorageAdapter implements StorageAdapter { } }); await conn.tx('set-indexes-with-schema-format', async t => { - if (insertedIndexes.length > 0) { - await self.createIndexes(className, insertedIndexes, t); + try { + if (insertedIndexes.length > 0) { + await self.createIndexes(className, insertedIndexes, t); + } + } catch (e) { + const columnDoesNotExistError = e.errors?.[0]?.code === '42703'; + if (columnDoesNotExistError && !this.disableIndexFieldValidation) { + throw e; + } } if (deletedIndexes.length > 0) { await self.dropIndexes(className, deletedIndexes, t); From 3d905364585139349db7c14f7446cbb6304a989f Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Tue, 23 Aug 2022 15:59:58 +0200 Subject: [PATCH 2/6] fix: grid fs adapter option --- spec/GridFSBucketStorageAdapter.spec.js | 8 ++++++++ src/Adapters/Files/GridFSBucketAdapter.js | 1 + 2 files changed, 9 insertions(+) diff --git a/spec/GridFSBucketStorageAdapter.spec.js b/spec/GridFSBucketStorageAdapter.spec.js index 7ffdced2bb..174b881c9f 100644 --- a/spec/GridFSBucketStorageAdapter.spec.js +++ b/spec/GridFSBucketStorageAdapter.spec.js @@ -405,6 +405,14 @@ describe_only_db('mongo')('GridFSBucket', () => { expect(gfsResult.toString('utf8')).toBe(twoMegabytesFile); }); + it('properly upload a file when disableIndexFieldValidation exist in databaseOptions', async () => { + const gfsAdapter = new GridFSBucketAdapter(databaseURI, { disableIndexFieldValidation: true }); + const twoMegabytesFile = randomString(2048 * 1024); + const res = await gfsAdapter.createFile('myFileName', twoMegabytesFile); + expect(res._id).toBeTruthy(); + expect(res.filename).toEqual('myFileName'); + }); + it('properly deletes a file from GridFS', async () => { const gfsAdapter = new GridFSBucketAdapter(databaseURI); await gfsAdapter.createFile('myFileName', 'a simple file'); diff --git a/src/Adapters/Files/GridFSBucketAdapter.js b/src/Adapters/Files/GridFSBucketAdapter.js index 06896e73b5..6b8f376e21 100644 --- a/src/Adapters/Files/GridFSBucketAdapter.js +++ b/src/Adapters/Files/GridFSBucketAdapter.js @@ -35,6 +35,7 @@ export class GridFSBucketAdapter extends FilesAdapter { useUnifiedTopology: true, }; this._mongoOptions = Object.assign(defaultMongoOptions, mongoOptions); + delete this._mongoOptions.disableIndexFieldValidation; } _connect() { From ae8cbee9e6715af1965773364ecf146f6dd1e0df Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Tue, 15 Nov 2022 00:05:54 +0100 Subject: [PATCH 3/6] fix: lint --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index b5132d21d7..a9d1156359 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1001,7 +1001,7 @@ export class PostgresStorageAdapter implements StorageAdapter { await self.createIndexes(className, insertedIndexes, t); } } catch (e) { - const columnDoesNotExistError = e.errors?.[0]?.code === '42703'; + const columnDoesNotExistError = e.errors && e.errors[0] && e.errors[0].code === '42703'; if (columnDoesNotExistError && !this.disableIndexFieldValidation) { throw e; } From 3647128964011c7ae6adbaaf5947471322347882 Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Sat, 13 Sep 2025 14:21:23 +0200 Subject: [PATCH 4/6] test: fix --- spec/GridFSBucketStorageAdapter.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/GridFSBucketStorageAdapter.spec.js b/spec/GridFSBucketStorageAdapter.spec.js index 25acc4a3b2..d30415edf3 100644 --- a/spec/GridFSBucketStorageAdapter.spec.js +++ b/spec/GridFSBucketStorageAdapter.spec.js @@ -424,9 +424,9 @@ describe_only_db('mongo')('GridFSBucket', () => { it('properly upload a file when disableIndexFieldValidation exist in databaseOptions', async () => { const gfsAdapter = new GridFSBucketAdapter(databaseURI, { disableIndexFieldValidation: true }); const twoMegabytesFile = randomString(2048 * 1024); - const res = await gfsAdapter.createFile('myFileName', twoMegabytesFile); - expect(res._id).toBeTruthy(); - expect(res.filename).toEqual('myFileName'); + await gfsAdapter.createFile('myFileName', twoMegabytesFile); + const gfsResult = await gfsAdapter.getFileData('myFileName'); + expect(gfsResult.toString('utf8')).toBe(twoMegabytesFile); }); it('properly deletes a file from GridFS', async () => { From 1ca3a35291fb336494b3e4409c01960634748cfa Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Wed, 8 Oct 2025 21:34:24 +0200 Subject: [PATCH 5/6] fix: error shallowing --- .../Postgres/PostgresStorageAdapter.js | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index c7ca77b639..f96905f03b 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -627,13 +627,11 @@ const buildWhereClause = ({ schema, query, index, caseInsensitive }): WhereClaus const distance = fieldValue.$maxDistance; const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${ - index + 2 + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2 })::geometry) <= $${index + 3}` ); sorts.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${ - index + 2 + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2 })::geometry) ASC` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); @@ -681,8 +679,7 @@ const buildWhereClause = ({ schema, query, index, caseInsensitive }): WhereClaus } const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${ - index + 2 + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2 })::geometry) <= $${index + 3}` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); @@ -877,7 +874,7 @@ export class PostgresStorageAdapter implements StorageAdapter { const { client, pgp } = createClient(uri, options); this._client = client; - this._onchange = () => {}; + this._onchange = () => { }; this._pgp = pgp; this._uuid = uuidv4(); this.canSortOnJoinTables = false; @@ -1018,7 +1015,13 @@ export class PostgresStorageAdapter implements StorageAdapter { } } catch (e) { const columnDoesNotExistError = e.errors && e.errors[0] && e.errors[0].code === '42703'; - if (columnDoesNotExistError && !this.disableIndexFieldValidation) { + // Specific case when the column does not exist + if (columnDoesNotExistError) { + // If the disableIndexFieldValidation is true, we should ignore the error + if (!this.disableIndexFieldValidation) { + throw e; + } + } else { throw e; } } @@ -1638,16 +1641,14 @@ export class PostgresStorageAdapter implements StorageAdapter { index += 2; } else if (fieldValue.__op === 'Remove') { updatePatterns.push( - `$${index}:name = array_remove(COALESCE($${index}:name, '[]'::jsonb), $${ - index + 1 + `$${index}:name = array_remove(COALESCE($${index}:name, '[]'::jsonb), $${index + 1 }::jsonb)` ); values.push(fieldName, JSON.stringify(fieldValue.objects)); index += 2; } else if (fieldValue.__op === 'AddUnique') { updatePatterns.push( - `$${index}:name = array_add_unique(COALESCE($${index}:name, '[]'::jsonb), $${ - index + 1 + `$${index}:name = array_add_unique(COALESCE($${index}:name, '[]'::jsonb), $${index + 1 }::jsonb)` ); values.push(fieldName, JSON.stringify(fieldValue.objects)); @@ -1758,8 +1759,7 @@ export class PostgresStorageAdapter implements StorageAdapter { updateObject = `COALESCE($${index}:name, '{}'::jsonb)`; } updatePatterns.push( - `$${index}:name = (${updateObject} ${deletePatterns} ${incrementPatterns} || $${ - index + 1 + keysToDelete.length + `$${index}:name = (${updateObject} ${deletePatterns} ${incrementPatterns} || $${index + 1 + keysToDelete.length }::jsonb )` ); values.push(fieldName, ...keysToDelete, JSON.stringify(fieldValue)); @@ -2198,8 +2198,7 @@ export class PostgresStorageAdapter implements StorageAdapter { groupByFields.push(`"${source}"`); } columns.push( - `EXTRACT(${ - mongoAggregateToPostgres[operation] + `EXTRACT(${mongoAggregateToPostgres[operation] } FROM $${index}:name AT TIME ZONE 'UTC')::integer AS $${index + 1}:name` ); values.push(source, alias); From 55071e668616c7a4be4b79668cd4ca55bb728516 Mon Sep 17 00:00:00 2001 From: Antoine Cormouls Date: Wed, 8 Oct 2025 22:26:10 +0200 Subject: [PATCH 6/6] fix: error detection --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index f96905f03b..7eaafcbde2 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1014,7 +1014,8 @@ export class PostgresStorageAdapter implements StorageAdapter { await self.createIndexes(className, insertedIndexes, t); } } catch (e) { - const columnDoesNotExistError = e.errors && e.errors[0] && e.errors[0].code === '42703'; + // pg-promise use Batch error see https://github.com/vitaly-t/spex/blob/e572030f261be1a8e9341fc6f637e36ad07f5231/src/errors/batch.js#L59 + const columnDoesNotExistError = e.getErrors && e.getErrors()[0] && e.getErrors()[0].code === '42703'; // Specific case when the column does not exist if (columnDoesNotExistError) { // If the disableIndexFieldValidation is true, we should ignore the error