Add option to restrict datetime precision

modules/module-mongodb/src/replication/MongoRelation.ts

@@ -8,7 +8,9 @@ import {
   CustomSqliteValue,
   SqliteInputRow,
   SqliteInputValue,
-  DateTimeValue
+  DateTimeValue,
+  DateTimeSourceOptions,
+  TimeValuePrecision
 } from '@powersync/service-sync-rules';
 
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
@@ -69,7 +71,7 @@ export function toMongoSyncRulesValue(data: any): SqliteInputValue {
     return data.toHexString();
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new DateTimeValue(isoString);
+    return new DateTimeValue(isoString, undefined, mongoTimeOptions);
   } else if (data instanceof mongo.Binary) {
     return new Uint8Array(data.buffer);
   } else if (data instanceof mongo.Long) {
@@ -122,7 +124,7 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
     return data;
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new DateTimeValue(isoString).toSqliteValue(context);
+    return new DateTimeValue(isoString, undefined, mongoTimeOptions).toSqliteValue(context);
   } else if (data instanceof mongo.ObjectId) {
     return data.toHexString();
   } else if (data instanceof mongo.UUID) {
@@ -195,3 +197,8 @@ export async function createCheckpoint(
     await session.endSession();
   }
 }
+
+const mongoTimeOptions: DateTimeSourceOptions = {
+  subSecondPrecision: TimeValuePrecision.milliseconds,
+  defaultSubSecondPrecision: TimeValuePrecision.milliseconds
+};

modules/module-mongodb/test/src/mongo_test.test.ts

@@ -4,7 +4,8 @@ import {
   CompatibilityContext,
   CompatibilityEdition,
   SqliteInputRow,
-  SqlSyncRules
+  SqlSyncRules,
+  TimeValuePrecision
 } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 
@@ -555,11 +556,23 @@ bucket_definitions:
         noFraction: '2023-03-06 13:47:01.000Z'
       });
 
-      const newFormat = applyRowContext(row, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      const newFormat = applyRowContext(row, new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }));
       expect(newFormat).toMatchObject({
         fraction: '2023-03-06T13:47:01.123Z',
         noFraction: '2023-03-06T13:47:01.000Z'
       });
+
+      const reducedPrecisionFormat = applyRowContext(
+        row,
+        new CompatibilityContext({
+          edition: CompatibilityEdition.SYNC_STREAMS,
+          maxTimeValuePrecision: TimeValuePrecision.seconds
+        })
+      );
+      expect(reducedPrecisionFormat).toMatchObject({
+        fraction: '2023-03-06T13:47:01Z',
+        noFraction: '2023-03-06T13:47:01Z'
+      });
     } finally {
       await client.close();
     }

modules/module-mysql/src/common/mysql-to-sqlite.ts

@@ -132,13 +132,15 @@ export function toSQLiteRow(
         case mysql.Types.TIMESTAMP:
         case ADDITIONAL_MYSQL_TYPES.TIMESTAMP2:
           {
-            const date = row[key] as Date;
-            if (isNaN(date.getTime())) {
+            const formattedDate = row[key] as string;
+            const parsed = new Date(formattedDate);
+
+            if (isNaN(parsed.getTime())) {
               // Invalid dates, such as 2024-00-00.
               // we can't do anything meaningful with this, so just use null.
               result[key] = null;
             } else {
-              result[key] = date.toISOString();
+              result[key] = new sync_rules.DateTimeValue(formattedDate, parsed.toISOString(), mySqlDateTimeOptions);
             }
           }
           break;
@@ -241,3 +243,9 @@ export function toExpressionTypeFromMySQLType(mysqlType: string | undefined): Ex
       return ExpressionType.TEXT;
   }
 }
+
+// We can provide microsecond accuracy, but for backwards compatibility we only offer millisecond accuracy by default.
+const mySqlDateTimeOptions: sync_rules.DateTimeSourceOptions = {
+  subSecondPrecision: sync_rules.TimeValuePrecision.microseconds,
+  defaultSubSecondPrecision: sync_rules.TimeValuePrecision.milliseconds
+};

modules/module-mysql/src/replication/MySQLConnectionManager.ts

@@ -48,7 +48,9 @@ export class MySQLConnectionManager {
       host: this.options.hostname,
       port: this.options.port,
       user: this.options.username,
-      password: this.options.password
+      password: this.options.password,
+      // We want to avoid parsing date/time values to Date, because that drops sub-millisecond precision.
+      dateStrings: true
     });
 
     this.binlogListeners.push(listener);

modules/module-mysql/test/src/mysql-to-sqlite.test.ts

@@ -1,4 +1,4 @@
-import { SqliteInputRow, SqliteRow } from '@powersync/service-sync-rules';
+import { applyRowContext, CompatibilityContext, SqliteInputRow, SqliteRow } from '@powersync/service-sync-rules';
 import { afterAll, describe, expect, test } from 'vitest';
 import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js';
@@ -228,8 +228,14 @@
       year_col: 2023
     };
 
-    expect(databaseRows[0]).toMatchObject(expectedResult);
-    expect(replicatedRows[0]).toMatchObject(expectedResult);
+    expect(applyRowContext(databaseRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
+    expect(applyRowContext(replicatedRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
+    expect(applyRowContext(databaseRows[0], new CompatibilityContext({ edition: 2 }))).toMatchObject(expectedResult);
+    expect(applyRowContext(replicatedRows[0], new CompatibilityContext({ edition: 2 }))).toMatchObject(expectedResult);
   });
 
   test('Date types edge cases mappings', async () => {
@@ -264,8 +270,12 @@
       const replicatedRows = await getReplicatedRows(expectedResults.length);
 
       for (let i = 0; i < expectedResults.length; i++) {
-        expect(databaseRows[i]).toMatchObject(expectedResults[i]);
-        expect(replicatedRows[i]).toMatchObject(expectedResults[i]);
+        expect(applyRowContext(databaseRows[i], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+          expectedResults[i]
+        );
+        expect(applyRowContext(replicatedRows[i], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+          expectedResults[i]
+        );
       }
     } finally {
       connection.release;

modules/module-postgres/test/src/pg_test.test.ts

@@ -5,7 +5,8 @@ import {
   SqliteInputRow,
   DateTimeValue,
   TimeValue,
-  CompatibilityEdition
+  CompatibilityEdition,
+  TimeValuePrecision
 } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js';
@@ -166,9 +167,9 @@ VALUES(10, ARRAY['null']::TEXT[]);
     expect(transformed[2]).toMatchObject({
       id: 3n,
       date: '2023-03-06',
-      time: new TimeValue('15:47:00'),
-      timestamp: new DateTimeValue('2023-03-06T15:47:00.000000', '2023-03-06 15:47:00'),
-      timestamptz: new DateTimeValue('2023-03-06T13:47:00.000000Z', '2023-03-06 13:47:00Z')
+      time: TimeValue.parse('15:47:00', pgwire.postgresTimeOptions),
+      timestamp: new DateTimeValue('2023-03-06T15:47:00.000000', '2023-03-06 15:47:00', pgwire.postgresTimeOptions),
+      timestamptz: new DateTimeValue('2023-03-06T13:47:00Z', '2023-03-06 13:47:00Z', pgwire.postgresTimeOptions)
     });
 
     expect(transformed[3]).toMatchObject({
@@ -183,26 +184,26 @@ VALUES(10, ARRAY['null']::TEXT[]);
     expect(transformed[4]).toMatchObject({
       id: 5n,
       date: '0000-01-01',
-      time: new TimeValue('00:00:00'),
-      timestamp: new DateTimeValue('0000-01-01T00:00:00'),
-      timestamptz: new DateTimeValue('0000-01-01T00:00:00Z')
+      time: TimeValue.parse('00:00:00', pgwire.postgresTimeOptions),
+      timestamp: new DateTimeValue('0000-01-01T00:00:00', undefined, pgwire.postgresTimeOptions),
+      timestamptz: new DateTimeValue('0000-01-01T00:00:00Z', undefined, pgwire.postgresTimeOptions)
     });
 
     expect(transformed[5]).toMatchObject({
       id: 6n,
-      timestamp: new DateTimeValue('1970-01-01T00:00:00.000000', '1970-01-01 00:00:00'),
-      timestamptz: new DateTimeValue('1970-01-01T00:00:00.000000Z', '1970-01-01 00:00:00Z')
+      timestamp: new DateTimeValue('1970-01-01T00:00:00.000000', '1970-01-01 00:00:00', pgwire.postgresTimeOptions),
+      timestamptz: new DateTimeValue('1970-01-01T00:00:00Z', '1970-01-01 00:00:00Z', pgwire.postgresTimeOptions)
     });
 
     expect(transformed[6]).toMatchObject({
       id: 7n,
-      timestamp: new DateTimeValue('9999-12-31T23:59:59'),
-      timestamptz: new DateTimeValue('9999-12-31T23:59:59Z')
+      timestamp: new DateTimeValue('9999-12-31T23:59:59', undefined, pgwire.postgresTimeOptions),
+      timestamptz: new DateTimeValue('9999-12-31T23:59:59Z', undefined, pgwire.postgresTimeOptions)
     });
 
     expect(transformed[7]).toMatchObject({
       id: 8n,
-      timestamptz: new DateTimeValue('0022-02-03T09:13:14.000000Z', '0022-02-03 09:13:14Z')
+      timestamptz: new DateTimeValue('0022-02-03T09:13:14Z', '0022-02-03 09:13:14Z', pgwire.postgresTimeOptions)
     });
 
     expect(transformed[8]).toMatchObject({
@@ -461,12 +462,25 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
         timestamptz: '2023-03-06 13:47:00Z'
       });
 
-      const newFormat = applyRowContext(row, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      const newFormat = applyRowContext(row, new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }));
       expect(newFormat).toMatchObject({
         time: '17:42:01.120000',
         timestamp: '2023-03-06T15:47:12.400000',
         timestamptz: '2023-03-06T13:47:00.000000Z'
       });
+
+      const reducedPrecisionFormat = applyRowContext(
+        row,
+        new CompatibilityContext({
+          edition: CompatibilityEdition.SYNC_STREAMS,
+          maxTimeValuePrecision: TimeValuePrecision.milliseconds
+        })
+      );
+      expect(reducedPrecisionFormat).toMatchObject({
+        time: '17:42:01.120',
+        timestamp: '2023-03-06T15:47:12.400',
+        timestamptz: '2023-03-06T13:47:00.000Z'
+      });
     } finally {
       await db.end();
     }
@@ -535,7 +549,10 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
         mood: 'happy'
       });
 
-      const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      const newFormat = applyRowContext(
+        transformed,
+        new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS })
+      );
       expect(newFormat).toMatchObject({
         rating: 1,
         composite: '{"foo":[2.0,3.0],"bar":"bar"}',
@@ -601,7 +618,10 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
         ranges: '{"{[2,4),[6,8)}"}'
       });
 
-      const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      const newFormat = applyRowContext(
+        transformed,
+        new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS })
+      );
       expect(newFormat).toMatchObject({
         ranges: JSON.stringify([
           [

modules/module-postgres/test/src/types/registry.test.ts

@@ -21,7 +21,7 @@ describe('custom type registry', () => {
 
     expect(applyValueContext(syncRulesValue, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toStrictEqual(old);
     expect(
-      applyValueContext(syncRulesValue, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS))
+      applyValueContext(syncRulesValue, new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }))
     ).toStrictEqual(fixed);
   }
 

packages/jpgwire/src/pgwire_types.ts

@@ -1,7 +1,12 @@
 // Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
 
 import { JsonContainer } from '@powersync/service-jsonbig';
-import { type DatabaseInputValue, TimeValue } from '@powersync/service-sync-rules';
+import {
+  type DatabaseInputValue,
+  DateTimeSourceOptions,
+  TimeValue,
+  TimeValuePrecision
+} from '@powersync/service-sync-rules';
 import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js';
 import { StructureParser } from './structure_parser.js';
 
@@ -134,7 +139,7 @@ export class PgType {
       case PgTypeOid.TIMESTAMPTZ:
         return timestamptzToSqlite(text);
       case PgTypeOid.TIME:
-        return TimeValue.parse(text);
+        return TimeValue.parse(text, postgresTimeOptions);
       case PgTypeOid.JSON:
       case PgTypeOid.JSONB:
         // Don't parse the contents
@@ -173,3 +178,8 @@ export class PgType {
     return '\\x' + Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
   }
 }
+
+export const postgresTimeOptions: DateTimeSourceOptions = Object.freeze({
+  subSecondPrecision: TimeValuePrecision.microseconds,
+  defaultSubSecondPrecision: TimeValuePrecision.microseconds
+});

packages/jpgwire/src/util.ts

@@ -3,7 +3,7 @@ import * as net from 'node:net';
 import * as tls from 'node:tls';
 import { DEFAULT_CERTS } from './certs.js';
 import * as pgwire from './pgwire.js';
-import { PgType } from './pgwire_types.js';
+import { PgType, postgresTimeOptions } from './pgwire_types.js';
 import { ConnectOptions } from './socket_adapter.js';
 import { DatabaseInputValue, DateTimeValue } from '@powersync/service-sync-rules';
 
@@ -243,9 +243,9 @@ export function timestamptzToSqlite(source?: string): DateTimeValue | null {
   const match = timeRegex.exec(source);
   if (match == null) {
     if (source == 'infinity') {
-      return new DateTimeValue('9999-12-31T23:59:59Z');
+      return new DateTimeValue('9999-12-31T23:59:59Z', undefined, postgresTimeOptions);
     } else if (source == '-infinity') {
-      return new DateTimeValue('0000-01-01T00:00:00Z');
+      return new DateTimeValue('0000-01-01T00:00:00Z', undefined, postgresTimeOptions);
     } else {
       return null;
     }
@@ -262,13 +262,12 @@ export function timestamptzToSqlite(source?: string): DateTimeValue | null {
 
   const baseValue = parsed.toISOString().replace('.000', '').replace('Z', '');
 
-  // In the new format, we always use ISO 8601. Since Postgres drops zeroes from the fractional seconds, we also pad
-  // that back to the highest theoretical precision (microseconds). This ensures that sorting returned values as text
-  // returns them in order of the time value they represent.
-  //
   // In the old format, we keep the sub-second precision only if it's not `.000`.
-  const missingPrecision = precision?.padEnd(7, '0') ?? '.000000';
-  return new DateTimeValue(`${baseValue}${missingPrecision}Z`, `${baseValue.replace('T', ' ')}${precision ?? ''}Z`);
+  return new DateTimeValue(
+    `${baseValue}${precision ?? ''}Z`,
+    `${baseValue.replace('T', ' ')}${precision ?? ''}Z`,
+    postgresTimeOptions
+  );
 }
 
 /**
@@ -286,9 +285,9 @@ export function timestampToSqlite(source?: string): DateTimeValue | null {
   const match = timeRegex.exec(source);
   if (match == null) {
     if (source == 'infinity') {
-      return new DateTimeValue('9999-12-31T23:59:59');
+      return new DateTimeValue('9999-12-31T23:59:59', undefined, postgresTimeOptions);
     } else if (source == '-infinity') {
-      return new DateTimeValue('0000-01-01T00:00:00');
+      return new DateTimeValue('0000-01-01T00:00:00', undefined, postgresTimeOptions);
     } else {
       return null;
     }
@@ -297,7 +296,7 @@ export function timestampToSqlite(source?: string): DateTimeValue | null {
   const [_, date, time, precision, __] = match as any;
   const missingPrecision = precision?.padEnd(7, '0') ?? '.000000';
 
-  return new DateTimeValue(`${date}T${time}${missingPrecision}`, source);
+  return new DateTimeValue(`${date}T${time}${missingPrecision}`, source, postgresTimeOptions);
 }
 /**
  * For date, we keep it mostly as-is.