Add support for quoted-printable encoded vCard files, fix #1513

package.json

@@ -9,9 +9,9 @@
 	"scripts": {
 		"flow": "flow; test $? -eq 0 -o $? -eq 2",
 		"start": "./start-desktop.sh",
-		"test": "cd test && node test api && node test client",
-		"testapi": "cd test && node test api",
-		"testclient": "cd test && node test client",
+		"test": "cd test && node --icu-data-dir=../node_modules/full-icu test api && node --icu-data-dir=../node_modules/full-icu test client",
+		"testapi": "cd test && node --icu-data-dir=../node_modules/full-icu test api",
+		"testclient": "cd test && node --icu-data-dir=../node_modules/full-icu test client",
 		"postinstall": "electron-builder install-app-deps"
 	},
 	"dependencies": {
@@ -48,6 +48,7 @@
 		"express": "4.17.1",
 		"flow-bin": "0.135.0 ",
 		"fs-extra": "8.1.0",
+		"full-icu": "^1.3.1",
 		"glob": "7.1.6",
 		"js-yaml": "3.13.1",
 		"mithril-node-render": "2.3.0",

src/api/common/utils/Encoding.js

@@ -315,3 +315,55 @@ export function base64ToUint8Array(base64: Base64): Uint8Array {
 	}
 	return result
 }
+
+/**
+ * Converts a Uint8Array containing string data into a string, given the charset the data is in.
+ * @param charset The charset. Must be supported by TextDecoder.
+ * @param bytes The string data
+ * @trhows RangeError if the charset is not supported
+ * @return The string
+ */
+export function uint8ArrayToString(charset: string, bytes: Uint8Array): string {
+	// $FlowExpectedError[incompatible-call] we will rely on the constructor throwing an error if the charset is not supported
+	const decoder = new TextDecoder(charset)
+	return decoder.decode(bytes);
+}
+
+/**
+ * Decodes a quoted-printable piece of text in a given charset.
+ * This was copied and modified from https://github.com/mathiasbynens/quoted-printable/blob/master/src/quoted-printable.js (MIT licensed)
+ *
+ * @param charset Must be supported by TextEncoder
+ * @param input The encoded text
+ * @throws RangeError if the charset is not supported
+ * @returns The text as a JavaScript string
+ */
+export function decodeQuotedPrintable(charset: string, input: string): string {
+	return input
+		// https://tools.ietf.org/html/rfc2045#section-6.7, rule 3:
+		// “Therefore, when decoding a `Quoted-Printable` body, any trailing white
+		// space on a line must be deleted, as it will necessarily have been added
+		// by intermediate transport agents.”
+		.replace(/[\t\x20]$/gm, '')
+		// Remove hard line breaks preceded by `=`. Proper `Quoted-Printable`-
+		// encoded data only contains CRLF line  endings, but for compatibility
+		// reasons we support separate CR and LF too.
+		.replace(/=(?:\r\n?|\n|$)/g, '')
+		// Decode escape sequences of the form `=XX` where `XX` is any
+		// combination of two hexidecimal digits. For optimal compatibility,
+		// lowercase hexadecimal digits are supported as well. See
+		// https://tools.ietf.org/html/rfc2045#section-6.7, note 1.
+		.replace(/(=([a-fA-F0-9]{2}))+/g,
+			match => {
+				const hexValues = match.split(/=/)
+				// splitting on '=' is convenient, but adds an empty string at the start due to the first byte
+				hexValues.shift()
+				const intArray = hexValues.map(char => parseInt(char, 16))
+				const bytes = Uint8Array.from(intArray)
+				return uint8ArrayToString(charset, bytes)
+			})
+}
+
+export function decodeBase64(charset: string, input: string): string {
+	return uint8ArrayToString(charset, base64ToUint8Array(input))
+}

src/contacts/VCardImporter.js

@@ -1,4 +1,5 @@
 // @flow
+import type {Contact} from "../api/entities/tutanota/Contact"
 import {createContact} from "../api/entities/tutanota/Contact"
 import {createContactAddress} from "../api/entities/tutanota/ContactAddress"
 import type {ContactAddressTypeEnum, ContactPhoneNumberTypeEnum} from "../api/common/TutanotaConstants"
@@ -9,7 +10,7 @@ import {createContactSocialId} from "../api/entities/tutanota/ContactSocialId"
 import {assertMainOrNode} from "../api/Env"
 import {createBirthday} from "../api/entities/tutanota/Birthday"
 import {birthdayToIsoDate} from "../api/common/utils/BirthdayUtils"
-import type {Contact} from "../api/entities/tutanota/Contact"
+import {decodeBase64, decodeQuotedPrintable} from "../api/common/utils/Encoding"
 
 assertMainOrNode()
 
@@ -24,7 +25,6 @@ export function vCardFileToVCards(vCardFileData: string): ?string[] {
 	vCardFileData = vCardFileData.replace(/begin:vcard/g, "BEGIN:VCARD")
 	vCardFileData = vCardFileData.replace(/end:vcard/g, "END:VCARD")
 	vCardFileData = vCardFileData.replace(/version:2.1/g, "VERSION:2.1")
-	let vCardList = []
 	if (vCardFileData.indexOf("BEGIN:VCARD") > -1
 		&& vCardFileData.indexOf(E) > -1
 		&& (vCardFileData.indexOf(V3) > -1 || vCardFileData.indexOf(V2) > -1)) {
@@ -79,6 +79,19 @@ export function vCardEscapingSplitAdr(addressDetails: string): string[] {
 }
 
 
+function _decodeTag(encoding: string, charset: string, text: string): string {
+	let decoder = (cs, l) => l
+	switch (encoding.toLowerCase()) {
+		case 'quoted-printable:':
+			decoder = decodeQuotedPrintable
+			break
+		case 'base64:':
+			decoder = decodeBase64
+	}
+
+	return text.split(';').map((line) => decoder(charset, line)).join(';')
+}
+
 export function vCardListToContacts(vCardList: string[], ownerGroupId: Id): Contact[] {
 	let contacts = []
 	for (let i = 0; i < vCardList.length; i++) {
@@ -93,6 +106,15 @@ export function vCardListToContacts(vCardList: string[], ownerGroupId: Id): Cont
 			let tagAndTypeString = vCardLines[j].substring(0, indexAfterTag).toUpperCase()
 			let tagName = tagAndTypeString.split(";")[0]
 			let tagValue = vCardLines[j].substring(indexAfterTag + 1)
+
+			let encodingObj = vCardLines[j].split(';').find((line) => line.includes('ENCODING='))
+			let encoding = encodingObj ? encodingObj.split('=')[1] : ''
+
+			let charsetObj = vCardLines[j].split(';').find((line) => line.includes('CHARSET='))
+			let charset = charsetObj ? charsetObj.split('=')[1] : ''
+
+			tagValue = _decodeTag(encoding, charset, tagValue)
+
 			switch (tagName) {
 				case "N":
 					let nameDetails = vCardReescapingArray(vCardEscapingSplit(tagValue))

test/api/common/EncodingTest.js

@@ -9,7 +9,7 @@ import {
 	base64ToBase64Url,
 	base64ToHex,
 	base64ToUint8Array,
-	base64UrlToBase64,
+	base64UrlToBase64, decodeBase64, decodeQuotedPrintable,
 	generatedIdToTimestamp,
 	hexToBase64,
 	hexToUint8Array,
@@ -18,9 +18,19 @@ import {
 	timestampToHexGeneratedId,
 	uint8ArrayToArrayBuffer,
 	uint8ArrayToBase64,
-	uint8ArrayToHex
+	uint8ArrayToHex, uint8ArrayToString
 } from "../../../src/api/common/utils/Encoding"
 import {GENERATED_MIN_ID} from "../../../src/api/common/EntityFunctions"
+// $FlowIssue[missing-export] TextEncoder *is* present in util.
+import {TextDecoder as nodeTextDecoder, TextEncoder as nodeTextEncoder} from "util"
+
+if (global.isBrowser) {
+	global.TextDecoder = window.TextDecoder
+	global.TextEncoder = window.TextEncoder
+} else {
+	global.TextDecoder = nodeTextDecoder
+	global.TextEncoder = nodeTextEncoder
+}
 
 o.spec("Encoding", function () {
 
@@ -155,5 +165,18 @@ o.spec("Encoding", function () {
 		o(Array.from(new Uint8Array(uint8ArrayToArrayBuffer(array.subarray(2))))).deepEquals([3, 4, 5])
 	})
 
+	o("uint8Array to string", function () {
+		o(uint8ArrayToString("utf-8", stringToUtf8Uint8Array("däß ißt ein teßt ü"))).equals("däß ißt ein teßt ü")
+		o(uint8ArrayToString("latin1", Uint8Array.from(["DC", "E7", "F1"].map(e => parseInt(e, 16))))).equals("Üçñ")
+	})
+
+	o("decode quoted-printable string", function () {
+		o(decodeQuotedPrintable("utf-8", "d=C3=A4=C3=9F i=C3=9Ft ein te=C3=9Ft =C3=BC")).equals("däß ißt ein teßt ü")
+		o(decodeQuotedPrintable("latin1", "Rua das Na=E7=F5es")).equals("Rua das Nações")
+	})
+	o("decode base64 string with utf8 charset", function () {
+		o(decodeBase64("utf-8", "ZMOkw58gacOfdCBlaW4gdGXDn3Qgw7w=")).equals("däß ißt ein teßt ü")
+		o(decodeBase64("latin1", "ZOTfIGnfdCBlaW4gdGXfdCD8")).equals("däß ißt ein teßt ü")
+	})
 
 })

test/client/contact/VCardImporterTest.js

@@ -9,6 +9,9 @@ import en from "../../../src/translations/en"
 import {lang} from "../../../src/misc/LanguageViewModel"
 import {ContactMailAddressTypeRef} from "../../../src/api/entities/tutanota/ContactMailAddress"
 import {ContactPhoneNumberTypeRef} from "../../../src/api/entities/tutanota/ContactPhoneNumber"
+import {TextDecoder} from "util"
+
+global.TextDecoder = global.isBrowser ? window.TextDecoder : TextDecoder;
 
 o.spec("VCardImporterTest", function () {
 
@@ -300,6 +303,49 @@ END:VCARD`
 		let contacts = vCardListToContacts(neverNull(vCardFileToVCards(vcards)), "")
 		o(neverNull(contacts[0].birthdayIso)).equals("--03-31")
 		o(neverNull(contacts[1].birthdayIso)).equals("--03-31")
+	});
+
+	o("quoted printable utf-8 entirely encoded", function () {
+		let vcards = "BEGIN:VCARD\n"
+			+ "VERSION:2.1\n"
+			+ "N:Mustermann;Max;;;\n"
+			+ "FN:Max Mustermann\n"
+			+ "ADR;HOME;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:;;=54=65=73=74=73=74=72=61=C3=9F=65=20=34=32;;;;\n"
+			+ "END:VCARD"
+		let contacts = vCardListToContacts(neverNull(vCardFileToVCards(vcards)), "")
+		o(neverNull(contacts[0].addresses[0].address)).equals("Teststraße 42")
+	})
+
+	o("quoted printable utf-8 partially encoded", function () {
+		let vcards = "BEGIN:VCARD\n"
+			+ "VERSION:2.1\n"
+			+ "N:Mustermann;Max;;;\n"
+			+ "FN:Max Mustermann\n"
+			+ "ADR;HOME;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:;;Teststra=C3=9Fe 42;;;;\n"
+			+ "END:VCARD"
+		let contacts = vCardListToContacts(neverNull(vCardFileToVCards(vcards)), "")
+		o(neverNull(contacts[0].addresses[0].address)).equals("Teststraße 42")
 	})
-})
 
+	o("base64 utf-8", function () {
+		let vcards = "BEGIN:VCARD\n"
+			+ "VERSION:2.1\n"
+			+ "N:Mustermann;Max;;;\n"
+			+ "FN:Max Mustermann\n"
+			+ "ADR;HOME;CHARSET=UTF-8;ENCODING=BASE64:;;w4TDpMOkaGhtbQ==;;;;\n"
+			+ "END:VCARD"
+		let contacts = vCardListToContacts(neverNull(vCardFileToVCards(vcards)), "")
+		o(neverNull(contacts[0].addresses[0].address)).equals("Ääähhmm")
+	})
+
+	o("test with latin charset", function () {
+		let vcards = "BEGIN:VCARD\n"
+			+ "VERSION:2.1\n"
+			+ "N:Mustermann;Max;;;\n"
+			+ "FN:Max Mustermann\n"
+			+ "ADR;HOME;CHARSET=ISO-8859-1;ENCODING=QUOTED-PRINTABLE:;;Rua das Na=E7=F5es;;;;\n"
+			+ "END:VCARD"
+		let contacts = vCardListToContacts(neverNull(vCardFileToVCards(vcards)), "")
+		o(neverNull(contacts[0].addresses[0].address)).equals("Rua das Nações")
+	})
+})

third-party.txt

@@ -102,4 +102,8 @@ Reference: http://plugins.cordova.io/#/package/org.apache.cordova.vibration
 
 JSZip - https://stuk.github.io/jszip/
 License: MIT
-Reference: https://github.com/Stuk/jszip/blob/master/LICENSE.markdown
+Reference: https://github.com/Stuk/jszip/blob/master/LICENSE.markdown
+
+quoted-printable - https://github.com/mathiasbynens/quoted-printable
+License: MIT
+Reference: https://github.com/mathiasbynens/quoted-printable/blob/master/LICENSE-MIT.txt