[#774] feat: read opreturn data as utf8

.env

@@ -8,5 +8,5 @@ MAIN_DB_PORT='3306'
 SUPERTOKENS_DB_USER='st-user'
 SUPERTOKENS_DB_PASSWORD='st-password'
 
-DATABASE_URL="mysql://$MAIN_DB_USER:$MAIN_DB_PASSWORD@$MAIN_DB_HOST:$MAIN_DB_PORT/$MAIN_DB_NAME"
-SHADOW_DATABASE_URL="mysql://$MAIN_DB_USER:$MAIN_DB_PASSWORD@$MAIN_DB_HOST:$MAIN_DB_PORT/prisma-shadow-db"
+DATABASE_URL="mysql://$MAIN_DB_USER:$MAIN_DB_PASSWORD@$MAIN_DB_HOST:$MAIN_DB_PORT/$MAIN_DB_NAME?charset=utf8mb4"
+SHADOW_DATABASE_URL="mysql://$MAIN_DB_USER:$MAIN_DB_PASSWORD@$MAIN_DB_HOST:$MAIN_DB_PORT/prisma-shadow-db?charset=utf8mb4"

docker-compose.yml

@@ -35,6 +35,7 @@ services:
     ports:
       - 3306:3306
     volumes:
+      - ./scripts/db/mysql-config.cnf:/etc/mysql/conf.d/mysql-config.cnf
       - ./scripts/db-entrypoint.sh:/docker-entrypoint-initdb.d/1-db-entrypoint.sh
       - ./scripts/db/init-test-db.sql:/home/mysql/raw_entrypoint/2-init-test-db.sql
       - ./scripts/db/init-supertokens-db.sql:/home/mysql/raw_entrypoint/3-init-supertokens-db.sql

scripts/db/mysql-config.cnf

@@ -0,0 +1,7 @@
+[mysqld]
+character-set-client-handshake = FALSE
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci
+
+[client]
+default-character-set = utf8mb4

services/chronikService.ts

@@ -16,6 +16,8 @@ import moment from 'moment'
 import { OpReturnData, parseError, parseOpReturnData } from 'utils/validators'
 import { executeAddressTriggers } from './triggerService'
 
+const decoder = new TextDecoder()
+
 interface ProcessedMessages {
   confirmed: KeyValueT<number>
   unconfirmed: KeyValueT<number>
@@ -61,14 +63,11 @@ export function getNullDataScriptData (outputScript: string): OpReturnData | nul
     return null
   }
 
-  let dataString = ''
-  for (let i = 0; i < dataPushData; i++) {
-    const hexByte = outputScript.slice(dataStartIndex + (i * 2), dataStartIndex + (i * 2) + 2)
-    const byteChar = String.fromCharCode(
-      parseInt(hexByte, 16)
-    )
-    dataString += byteChar
-  }
+  const dataHexBuffer = Buffer.from(
+    outputScript.slice(dataStartIndex, dataStartIndex + dataPushData * 2),
+    'hex'
+  )
+  const dataString = decoder.decode(dataHexBuffer)
 
   const ret: OpReturnData = {
     data: parseOpReturnData(dataString),

tests/unittests/chronikService.test.ts

@@ -42,23 +42,23 @@ describe('getNullDataScriptData tests', () => {
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual(null)
   })
-  it('Simple string data', async () => {
+  it('String data', async () => {
     const script = '6a' + '04' + '50415900' + '00' + '08' + '5051525354555657'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({
       paymentId: '',
       data: 'PQRSTUVW'
     })
   })
-  it('Simple array data', async () => {
+  it('Array data', async () => {
     const script = '6a' + '04' + '50415900' + '00' + '0b' + '6974656d317c6974656d32'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({
       paymentId: '',
       data: ['item1', 'item2']
     })
   })
-  it('Simple dict data', async () => {
+  it('Dict data', async () => {
     const script = '6a' + '04' + '50415900' + '00' + '14' + '6b65793d76616c756520736f6d653d6f74686572'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({
@@ -69,7 +69,7 @@ describe('getNullDataScriptData tests', () => {
       }
     })
   })
-  it('Simple dict with array', async () => {
+  it('Dict with array', async () => {
     const script = '6a' + '04' + '50415900' + '00' + '1c' + '6b65793d76616c756520736f6d653d76616c7565317c76616c756532'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({
@@ -80,15 +80,37 @@ describe('getNullDataScriptData tests', () => {
       }
     })
   })
-  it('Simple string data with nonce', async () => {
-    const script = '6a' + '04' + '50415900' + '00' + '08' + '505152535455565703010203'
+  it('Non-ASCII data', async () => {
+    const script = (
+      '6a' + '04' + '50415900' + '00' + '3e' +
+      'f09f9882f09f918dc2a9c4b8c3b0d09cd0b6d0aad18b2520c58bc3a650c39fc491c4b8c582e2809ec2bbe2809cc3a67dc2b9e28693c2a3c2b3e28692c2b2'
+    )
+    const data = getNullDataScriptData(script)
+    expect(data).toStrictEqual({
+      paymentId: '',
+      data: '😂👍©ĸðМжЪы% ŋæPßđĸł„»“æ}¹↓£³→²'
+    })
+  })
+  it('Non-ASCII data with paymentId', async () => {
+    const script = ('6a' + '04' + '50415900' + '00' + '3e' +
+      'f09f9882f09f918dc2a9c4b8c3b0d09cd0b6d0aad18b2520c58bc3a650c39fc491c4b8c582e2809ec2bbe2809cc3a67dc2b9e28693c2a3c2b3e28692c2b2' +
+      '08' + 'ab192bcafd745acd'
+    )
+    const data = getNullDataScriptData(script)
+    expect(data).toStrictEqual({
+      paymentId: 'ab192bcafd745acd',
+      data: '😂👍©ĸðМжЪы% ŋæPßđĸł„»“æ}¹↓£³→²'
+    })
+  })
+  it('String data with nonce', async () => {
+    const script = '6a' + '04' + '50415900' + '00' + '08' + '5051525354555657' + '03' + '010203'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({
       paymentId: '010203',
       data: 'PQRSTUVW'
     })
   })
-  it('Simple string data with explicitly empty nonce', async () => {
+  it('String data with explicitly empty nonce', async () => {
     const script = '6a' + '04' + '50415900' + '00' + '08' + '5051525354555657' + '00'
     const data = getNullDataScriptData(script)
     expect(data).toStrictEqual({