Fix #459: reduce timestamptz precision (#716)

* fix #459: add a test to reproduce the issue

* fixes #459: reduce timestamptz precision across the database

lib/model/migrations/20221208-01-reduce-tz-precision.js

@@ -0,0 +1,46 @@
+// Copyright 2022 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+// Issue:        #cb459 - `gt` filter for submissionDate is not working as expected because of tz precision
+// Root cause:   Default timestamptz precision in postgres is microseconds and node/js has just milliseconds
+// Solution:     Let's change precision to milliseconds in database, since there is no value in having higher precision
+//               in the database when application can't use/handle it + typical usage of ODK Central doesn't demand higher
+//               precision.
+
+const getTimestampColumns = (db) => db.raw(`
+  SELECT
+    table_name, column_name, is_nullable
+  FROM
+    information_schema.columns
+  WHERE
+    table_schema = 'public'
+    AND udt_name = 'timestamptz'`)
+  .then(data => data.rows);
+
+const changePrecision = (db, columnMeta, precision) => db.schema.alterTable(columnMeta.table_name, (table) => {
+  if (columnMeta.is_nullable === 'YES') {
+    table.dateTime(columnMeta.column_name, { useTz: true, precision }).alter();
+  } else {
+    table.dateTime(columnMeta.column_name, { useTz: true, precision }).notNullable().alter();
+  }
+});
+
+const up = async (db) => {
+  const columns = await getTimestampColumns(db);
+
+  await Promise.all(columns.map(c => changePrecision(db, c, 3)));
+};
+
+const down = async (db) => {
+  const columns = await getTimestampColumns(db);
+
+  await Promise.all(columns.map(c => changePrecision(db, c, 6)));
+};
+
+module.exports = { up, down };

test/integration/api/odata.js

@@ -857,6 +857,36 @@ describe('api: /forms/:id.svc', () => {
               });
             })))));
 
+    // #cb459: `gt` filter for submissionDate is not working as expected because of tz precision
+    // This test fails without 20221208-01-reduce-tz-precision.js (before-after state)
+    it('should only return submissions with submissionDate gt provided timestamp', testService(async (service, { run }) => {
+      const asAlice = await service.login('alice', identity);
+
+      await asAlice.post('/v1/projects/1/forms/withrepeat/submissions')
+        .send(testData.instances.withrepeat.one)
+        .set('Content-Type', 'text/xml')
+        .expect(200);
+
+      // Ensure that microsecond does not ends at 000 - In that case existing code is working correctly
+      await run(sql`update submissions set "createdAt"=date_trunc('milliseconds', "createdAt") + interval '1 microseconds'`);
+
+      const lastTimestamp = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions')
+        .expect(200)
+        .then(({ body }) => body.value[0].__system.submissionDate);
+
+      await asAlice.post('/v1/projects/1/forms/withrepeat/submissions')
+        .send(testData.instances.withrepeat.two)
+        .set('Content-Type', 'text/xml')
+        .expect(200);
+
+      await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions?$filter=__system/submissionDate gt ${lastTimestamp}`)
+        .expect(200)
+        .then(({ body }) => {
+          body.value.length.should.be.eql(1);
+          body.value[0].__id.should.be.eql('rtwo');
+        });
+    }));
+
     it('should return submissionDate-filtered toplevel rows with a function', testService((service, { run }) =>
       service.login('alice', (asAlice) =>
         asAlice.post('/v1/projects/1/forms/withrepeat/submissions')