diff --git a/.changeset/spotty-buckets-flash.md b/.changeset/spotty-buckets-flash.md new file mode 100644 index 0000000..ff6ffb0 --- /dev/null +++ b/.changeset/spotty-buckets-flash.md @@ -0,0 +1,5 @@ +--- +'@powersync/mysql-zongji': minor +--- + +Export date and time values as structured fields. diff --git a/README.md b/README.md index ca6929a..fcf0efe 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,6 @@ The `ZongJi` constructor accepts one argument of either: If a `Connection` or `Pool` object is passed to the constructor, it will not be destroyed/ended by Zongji's `stop()` method. -If there is a `dateStrings` `mysql` configuration option in the connection details or connection, `ZongJi` will follow it. - Each instance includes the following methods: | Method Name | Arguments | Description | @@ -139,7 +137,6 @@ Neither method requires any arguments. - :star2: [All types allowed by `mysql`](https://github.com/mysqljs/mysql#type-casting) are supported by this package. - :speak_no_evil: 64-bit integer is supported via package big-integer(see #108). If an integer is within the safe range of JS number (-2^53, 2^53), a Number object will returned, otherwise, will return as String. - :point_right: `TRUNCATE` statement does not cause corresponding `DeleteRows` event. Use unqualified `DELETE FROM` for same effect. -- When using fractional seconds with `DATETIME` and `TIMESTAMP` data types in MySQL > 5.6.4, only millisecond precision is available due to the limit of Javascript's `Date` object. ## Run Tests diff --git a/lib/common.js b/lib/common.js index 6b4dbe2..8cb8133 100644 --- a/lib/common.js +++ b/lib/common.js @@ -1,6 +1,5 @@ const iconv = require('iconv-lite'); const decodeJson = require('./json_decode'); -const dtDecode = require('./datetime_decode'); const bigInt = require('big-integer'); const MysqlTypes = (exports.MysqlTypes = { @@ -332,27 +331,14 @@ const parseGeometryValue = function (buffer) { // Returns false, or an object describing the fraction of a second part of a // TIME, DATETIME, or TIMESTAMP. const readTemporalFraction = function (parser, fractionPrecision) { - if (!fractionPrecision) return false; + if (!fractionPrecision) return undefined; let fractionSize = Math.ceil(fractionPrecision / 2); let fraction = readIntBE(parser._buffer, parser._offset, fractionSize); parser._offset += fractionSize; if (fractionPrecision % 2 !== 0) fraction /= 10; // Not using full space if (fraction < 0) fraction *= -1; // Negative time, fraction not negative - let milliseconds; - if (fractionPrecision > 3) { - milliseconds = Math.floor(fraction / Math.pow(10, fractionPrecision - 3)); - } else if (fractionPrecision < 3) { - milliseconds = fraction * Math.pow(10, 3 - fractionPrecision); - } else { - milliseconds = fraction; - } - - return { - value: fraction, // the integer after the decimal place - precision: fractionPrecision, // the number of digits after the decimal - milliseconds: milliseconds // the unrounded 3 digits after the decimal - }; + return { fraction, precision: fractionPrecision }; }; // This function is used to read and interpret non-null values from parser. @@ -485,12 +471,12 @@ exports.readMysqlValue = function ( break; case MysqlTypes.DATE: raw = parseUInt24(parser); - result = dtDecode.getDate( - zongji.connection.config.dateStrings, // node-mysql dateStrings option - sliceBits(raw, 9, 24), // year - sliceBits(raw, 5, 9), // month - sliceBits(raw, 0, 5) // day - ); + + result = { + year: sliceBits(raw, 9, 24), + month: sliceBits(raw, 5, 9), + day: sliceBits(raw, 0, 5) + }; break; case MysqlTypes.TIME: raw = parseUInt24(parser); @@ -523,35 +509,31 @@ exports.readMysqlValue = function ( minute = sliceBits(raw, 6, 12); second = sliceBits(raw, 0, 6); - if (isNegative && (fraction === false || fraction.value === 0)) { + if (isNegative && (fraction === undefined || fraction.value === 0)) { second++; } - result = - (isNegative ? '-' : '') + - zeroPad(hour, hour > 99 ? 3 : 2) + - ':' + - zeroPad(minute, 2) + - ':' + - zeroPad(second, 2); + result = { + isNegative, + hour, + minute, + second, + fraction + }; - if (fraction !== false) { - result += dtDecode.getFractionString(fraction); - } break; case MysqlTypes.DATETIME: raw = parseUInt64(parser); date = Math.floor(raw / 1000000); time = raw % 1000000; - result = dtDecode.getDateTime( - zongji.connection.config.dateStrings, // node-mysql dateStrings option - Math.floor(date / 10000), // year - Math.floor((date % 10000) / 100), // month - date % 100, // day - Math.floor(time / 10000), // hour - Math.floor((time % 10000) / 100), // minutes - time % 100 // seconds - ); + result = { + year: Math.floor(date / 10000), + month: Math.floor((date % 10000) / 100), + day: date % 100, + hour: Math.floor(time / 10000), + minute: Math.floor((time % 10000) / 100), + second: time % 100 + }; break; case MysqlTypes.DATETIME2: { // Overlapping high-low to get all data in 32-bit numbers @@ -561,31 +543,30 @@ exports.readMysqlValue = function ( fraction = readTemporalFraction(parser, column.metadata.decimals); yearMonth = sliceBits(rawHigh, 14, 31); - result = dtDecode.getDateTime( - zongji.connection.config.dateStrings, // node-mysql dateStrings option - Math.floor(yearMonth / 13), // year - yearMonth % 13, // month - sliceBits(rawLow, 17, 22), // day - sliceBits(rawLow, 12, 17), // hour - sliceBits(rawLow, 6, 12), // minutes - sliceBits(rawLow, 0, 6), // seconds - fraction // fraction of a second object - ); + result = { + year: Math.floor(yearMonth / 13), + month: yearMonth % 13, + day: sliceBits(rawLow, 17, 22), + hour: sliceBits(rawLow, 12, 17), + minute: sliceBits(rawLow, 6, 12), + second: sliceBits(rawLow, 0, 6), + fraction + }; break; } case MysqlTypes.TIMESTAMP: raw = parser.parseUnsignedNumber(4); - result = dtDecode.getTimeStamp(zongji.connection.config.dateStrings, raw); + result = { + secondsFromEpoch: raw + }; break; case MysqlTypes.TIMESTAMP2: raw = readIntBE(parser._buffer, parser._offset, 4); parser._offset += 4; - fraction = readTemporalFraction(parser, column.metadata.decimals); - result = dtDecode.getTimeStamp( - zongji.connection.config.dateStrings, - raw, // seconds from epoch - fraction - ); // fraction of a second object + result = { + secondsFromEpoch: raw, + fraction: readTemporalFraction(parser, column.metadata.decimals) + }; break; case MysqlTypes.YEAR: raw = parser.parseUnsignedNumber(1); diff --git a/lib/datetime_decode.js b/lib/datetime_decode.js deleted file mode 100644 index c7bb56e..0000000 --- a/lib/datetime_decode.js +++ /dev/null @@ -1,112 +0,0 @@ -// This file contains functions useful for converting bare numbers into -// JavaScript Date objects, or DATE/DATETIME/TIMESTAMP strings, according to the -// dateStrings option. The dateStrings option should be read from -// zongji.connection.config, where zongji is the current instance of the ZongJi -// object. The dateStrings option is interpreted the same as in node-mysql. -const common = require('./common'); // used only for common.zeroPad - -// dateStrings are used only if the dateStrings option is true, or is an array -// containing the sql type name string, 'DATE', 'DATETIME', or 'TIMESTAMP'. -// This follows the documentation of the dateStrings option in node-mysql. -const useDateStringsForType = function (dateStrings, sqlTypeString) { - return dateStrings && (dateStrings === true || (dateStrings.indexOf && dateStrings.indexOf(sqlTypeString) > -1)); -}; - -// fraction is the fractional second object from readTemporalFraction(). -// returns '' or a '.' followed by fraction.precision digits, like '.123' -const getFractionString = (exports.getFractionString = function (fraction) { - return fraction ? '.' + common.zeroPad(fraction.value, fraction.precision) : ''; -}); - -// 1950-00-00 and the like are perfectly valid Mysql dateStrings. A 0 portion -// of a date is essentially a null part of the date, so we should keep it. -// year, month, and date must be integers >= 0. January is month === 1. -const getDateString = (exports.getDateString = function (year, month, date) { - return common.zeroPad(year, 4) + '-' + common.zeroPad(month, 2) + '-' + common.zeroPad(date, 2); -}); - -// Date object months are 1 less than Mysql months, and we need to filter 0. -// If we don't filter 0, 2017-00-01 will become the javascript Date 2016-12-01, -// which is not what it means. It means 2017-NULL-01, but the Date object -// cannot handle it, so we want to return an invalid month, rather than a -// subtracted month. -const jsMonthFromMysqlMonth = function (month) { - return month > 0 ? month - 1 : undefined; -}; - -// Returns a new Date object or Mysql dateString, following the dateStrings -// option. With the dateStrings option, it can output valid Mysql DATE strings -// representing values that cannot be represented by a Date object, such as -// values with a null part like '1950-00-04', or a zero date '0000-00-00'. -exports.getDate = function ( - dateStrings, // node-mysql dateStrings option - year, - month, // January === 1 - date -) { - if (!useDateStringsForType(dateStrings, 'DATE')) { - return new Date(Date.UTC(year, jsMonthFromMysqlMonth(month), date)); - } - return getDateString(year, month, date); -}; - -// Returns a new Date object or Mysql dateString, following the dateStrings -// option. Fraction is an optional parameter that comes from -// readTemporalFraction(). Mysql dateStrings are needed for microsecond -// precision, or to represent '0000-00-00 00:00:00'. -exports.getDateTime = function ( - dateStrings, // node-mysql dateStrings option - year, - month, // January === 1 - date, - hour, - minute, - second, - fraction // optional fractional second object -) { - if (!useDateStringsForType(dateStrings, 'DATETIME')) { - return new Date( - Date.UTC(year, jsMonthFromMysqlMonth(month), date, hour, minute, second, fraction ? fraction.milliseconds : 0) - ); - } - return ( - getDateString(year, month, date) + - ' ' + - common.zeroPad(hour, 2) + - ':' + - common.zeroPad(minute, 2) + - ':' + - common.zeroPad(second, 2) + - getFractionString(fraction) - ); -}; - -// Returns a new Date object or Mysql dateString, following the dateStrings -// option. Fraction is an optional parameter that comes from -// readTemporalFraction(). With the dateStrings option from node-mysql, -// this returns a mysql TIMESTAMP string, like '1975-03-01 23:03:20.38945' or -// '1975-03-01 00:03:20'. Mysql strings are needed for precision beyond ms. -exports.getTimeStamp = function ( - dateStrings, // node-mysql dateStrings option - secondsFromEpoch, // an integer - fraction // optional fraction of second object -) { - const milliseconds = fraction ? fraction.milliseconds : 0; - const dateObject = new Date(secondsFromEpoch * 1000 + milliseconds); - if (!useDateStringsForType(dateStrings, 'TIMESTAMP')) { - return dateObject; - } - if (secondsFromEpoch === 0 && (!fraction || fraction.value === 0)) { - return '0000-00-00 00:00:00' + getFractionString(fraction); - } - return ( - getDateString(dateObject.getFullYear(), dateObject.getMonth() + 1, dateObject.getDate()) + - ' ' + - common.zeroPad(dateObject.getHours(), 2) + - ':' + - common.zeroPad(dateObject.getMinutes(), 2) + - ':' + - common.zeroPad(dateObject.getSeconds(), 2) + - getFractionString(fraction) - ); -}; diff --git a/test/types.js b/test/types.js index 8f18323..e53ef36 100644 --- a/test/types.js +++ b/test/types.js @@ -68,32 +68,24 @@ function defineTypeTest(name, fields, testRows, customTest) { } }; - expectEvents( - test, - eventLog, - [ - expectedWrite - ], - testRows.length, - () => { - test.equal(errorLog.length, 0); + expectEvents(test, eventLog, [expectedWrite], testRows.length, () => { + test.equal(errorLog.length, 0); - const binlogRows = eventLog.reduce((prev, curr) => { - if (curr.getTypeName() === 'WriteRows') { - prev = prev.concat(curr.rows); - } - return prev; - }, []); - - if (customTest) { - customTest.bind(selectResult)(test, { rows: binlogRows }); - } else { - assert.deepEqual(selectResult, binlogRows); + const binlogRows = eventLog.reduce((prev, curr) => { + if (curr.getTypeName() === 'WriteRows') { + prev = prev.concat(curr.rows); } + return prev; + }, []); - test.end(); + if (customTest) { + customTest.bind(selectResult)(test, { rows: binlogRows }); + } else { + assert.deepEqual(selectResult, binlogRows); } - ); + + test.end(); + }); }); }); }); @@ -255,13 +247,77 @@ defineTypeTest( ["'01:27:28'"], ["'-01:07:08'"], ["'-01:27:28'"] - ] + ], + function (_, event) { + const values = event.rows.map((e) => e.col0); + assert.deepEqual(values, [ + { isNegative: true, hour: 0, minute: 0, second: 1, fraction: undefined }, + { isNegative: false, hour: 0, minute: 0, second: 0, fraction: undefined }, + { isNegative: false, hour: 0, minute: 7, second: 0, fraction: undefined }, + { isNegative: false, hour: 20, minute: 0, second: 0, fraction: undefined }, + { isNegative: false, hour: 19, minute: 0, second: 0, fraction: undefined }, + { isNegative: false, hour: 4, minute: 0, second: 0, fraction: undefined }, + { isNegative: true, hour: 838, minute: 59, second: 59, fraction: undefined }, + { isNegative: false, hour: 838, minute: 59, second: 59, fraction: undefined }, + { isNegative: false, hour: 1, minute: 7, second: 8, fraction: undefined }, + { isNegative: false, hour: 1, minute: 27, second: 28, fraction: undefined }, + { isNegative: true, hour: 1, minute: 7, second: 8, fraction: undefined }, + { isNegative: true, hour: 1, minute: 27, second: 28, fraction: undefined } + ]); + } +); + +defineTypeTest( + 'time_fraction', + ['TIME(3) NULL', 'DATETIME(6) NULL', 'TIMESTAMP(2) NULL'], + [["'17:51:04.777'", "'2018-09-08 17:51:04.777'", "'2018-09-08 17:51:04.777'"]], + function (_, event) { + assert.deepEqual(event.rows, [ + { + col0: { + isNegative: false, + hour: 17, + minute: 51, + second: 4, + fraction: { + fraction: 777, + precision: 3 + } + }, + col1: { + year: 2018, + month: 9, + day: 8, + hour: 17, + minute: 51, + second: 4, + fraction: { + fraction: 777000, + precision: 6 + } + }, + col2: { + fraction: { fraction: 78, precision: 2 }, + secondsFromEpoch: 1536429064 + } + } + ]); + } ); defineTypeTest( 'datetime_no_fraction', ['DATETIME NULL'], - [["'1000-01-01 00:00:00'"], ["'9999-12-31 23:59:59'"], ["'2014-12-27 01:07:08'"]] + [["'1000-01-01 00:00:00'"], ["'9999-12-31 23:59:59'"], ["'2014-12-27 01:07:08'"]], + function (_, event) { + assert.deepEqual(event.rows, [ + { col0: { year: 1000, month: 1, day: 1, hour: 0, minute: 0, second: 0, fraction: undefined } }, + { col0: { year: 9999, month: 12, day: 31, hour: 23, minute: 59, second: 59, fraction: undefined } }, + { + col0: { year: 2014, month: 12, day: 27, hour: 1, minute: 7, second: 8, fraction: undefined } + } + ]); + } ); defineTypeTest( @@ -271,7 +327,22 @@ defineTypeTest( ["'1000-01-01'", "'1970-01-01 00:00:01'", 1901], ["'9999-12-31'", "'2038-01-18 03:14:07'", 2155], ["'2014-12-27'", "'2014-12-27 01:07:08'", 2014] - ] + ], + function (_, event) { + assert.deepEqual(event.rows, [ + { col0: { year: 1000, month: 1, day: 1 }, col1: { secondsFromEpoch: 1, fraction: undefined }, col2: 1901 }, + { + col0: { day: 31, month: 12, year: 9999 }, + col1: { secondsFromEpoch: 2147397247, fraction: undefined }, + col2: 2155 + }, + { + col0: { year: 2014, month: 12, day: 27 }, + col1: { secondsFromEpoch: 1419642428, fraction: undefined }, + col2: 2014 + } + ]); + } ); defineTypeTest( diff --git a/types/index.d.ts b/types/index.d.ts index 9db762a..6dea1d0 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -15,7 +15,6 @@ export type ZongjiOptions = { port?: number; user: string; password: string; - dateStrings?: boolean; timeZone?: string; }; @@ -205,3 +204,35 @@ export declare class ZongJi extends EventEmitter { pause(): void; resume(): void; } + +export interface MySqlDate { + year: number; + month: number; + day: number; +} + +export interface MySqlTime { + isNegative: boolean; + hour: number; + minute: number; + second: number; + fraction?: MySqlFraction; +} + +export interface MySqlDateTime extends MySqlDate, MySqlTime {} + +export interface MySqlFraction { + /** + * The digits making up this fraction, e.g. `123` for `.123`, `0.0123`, `0.00123` and so on. + */ + fraction: number; + /** + * The amount of digits after the decimal. + */ + precision: number; +} + +export interface MySqlTimeStamp { + secondsFromEpoch: number; + fraction?: MySqlFraction; +}