diff --git a/lib/model/migrations/20221208-01-reduce-tz-precision.js b/lib/model/migrations/20221208-01-reduce-tz-precision.js new file mode 100644 index 000000000..9d5d4faba --- /dev/null +++ b/lib/model/migrations/20221208-01-reduce-tz-precision.js @@ -0,0 +1,46 @@ +// Copyright 2022 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +// Issue: #cb459 - `gt` filter for submissionDate is not working as expected because of tz precision +// Root cause: Default timestamptz precision in postgres is microseconds and node/js has just milliseconds +// Solution: Let's change precision to milliseconds in database, since there is no value in having higher precision +// in the database when application can't use/handle it + typical usage of ODK Central doesn't demand higher +// precision. + +const getTimestampColumns = (db) => db.raw(` + SELECT + table_name, column_name, is_nullable + FROM + information_schema.columns + WHERE + table_schema = 'public' + AND udt_name = 'timestamptz'`) + .then(data => data.rows); + +const changePrecision = (db, columnMeta, precision) => db.schema.alterTable(columnMeta.table_name, (table) => { + if (columnMeta.is_nullable === 'YES') { + table.dateTime(columnMeta.column_name, { useTz: true, precision }).alter(); + } else { + table.dateTime(columnMeta.column_name, { useTz: true, precision }).notNullable().alter(); + } +}); + +const up = async (db) => { + const columns = await getTimestampColumns(db); + + await Promise.all(columns.map(c => changePrecision(db, c, 3))); +}; + +const down = async (db) => { + const columns = await getTimestampColumns(db); + + await Promise.all(columns.map(c => changePrecision(db, c, 6))); +}; + +module.exports = { up, down }; diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 151b893a8..e7935a291 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -857,6 +857,36 @@ describe('api: /forms/:id.svc', () => { }); }))))); + // #cb459: `gt` filter for submissionDate is not working as expected because of tz precision + // This test fails without 20221208-01-reduce-tz-precision.js (before-after state) + it('should only return submissions with submissionDate gt provided timestamp', testService(async (service, { run }) => { + const asAlice = await service.login('alice', identity); + + await asAlice.post('/v1/projects/1/forms/withrepeat/submissions') + .send(testData.instances.withrepeat.one) + .set('Content-Type', 'text/xml') + .expect(200); + + // Ensure that microsecond does not ends at 000 - In that case existing code is working correctly + await run(sql`update submissions set "createdAt"=date_trunc('milliseconds', "createdAt") + interval '1 microseconds'`); + + const lastTimestamp = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions') + .expect(200) + .then(({ body }) => body.value[0].__system.submissionDate); + + await asAlice.post('/v1/projects/1/forms/withrepeat/submissions') + .send(testData.instances.withrepeat.two) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions?$filter=__system/submissionDate gt ${lastTimestamp}`) + .expect(200) + .then(({ body }) => { + body.value.length.should.be.eql(1); + body.value[0].__id.should.be.eql('rtwo'); + }); + })); + it('should return submissionDate-filtered toplevel rows with a function', testService((service, { run }) => service.login('alice', (asAlice) => asAlice.post('/v1/projects/1/forms/withrepeat/submissions')