Add case folding data

.gitignore

@@ -1,6 +1,5 @@
-# Generated data folders
-[0-9].[0-9].[0-9][0-9]
-[0-9].[0-9].[0-9]
+# Generated data
+output
 
 # JSON version of coverage report
 coverage/*.json

README.md

@@ -168,6 +168,9 @@ Note that these READMEs are auto-generated by this script, too – they describe
     'Variation Selectors Supplement', 'Supplementary Private Use Area-A',
     'Supplementary Private Use Area-B'
   ],
+  'case-folding': [
+    'C', 'T', 'F', 'S'
+  ],
   'bidi': [
     'AL', 'B', 'CS', 'ES', 'FSI', 'LRE', 'LRO', 'ON', ' PDI', 'RLE', 'RLO',
     'WS', 'AN', 'BN', 'EN', 'ET', 'L', 'LRI', 'NSM', 'PDF', 'R', 'RLI', 'S'

bootstrap.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "${BASH_SOURCE}")";
+
+node scripts/generate-data.js;
+
+cd output;
+
+for dir in *; do
+	cd "${dir}";
+	echo "Taking care of ${dir}…";
+	#npm publish;
+	cd ..;
+done;

data/resources.js

@@ -40,31 +40,35 @@ var resources = [
 		'scripts': 'http://unicode.org/Public/3.1-Update/Scripts-3.1.0.txt',
 		'blocks': 'http://unicode.org/Public/3.1-Update/Blocks-4.txt',
 		'properties': 'http://unicode.org/Public/3.1-Update/PropList-3.1.0.txt',
-		'derived-core-properties': 'http://unicode.org/Public/3.1-Update/DerivedCoreProperties-3.1.0.txt'
+		'derived-core-properties': 'http://unicode.org/Public/3.1-Update/DerivedCoreProperties-3.1.0.txt',
+		'case-folding': 'http://unicode.org/Public/3.1-Update/CaseFolding-3.txt'
 	},
 	{
 		'version': '3.2.0',
 		'main': 'http://unicode.org/Public/3.2-Update/UnicodeData-3.2.0.txt',
 		'scripts': 'http://unicode.org/Public/3.2-Update/Scripts-3.2.0.txt',
 		'blocks': 'http://unicode.org/Public/3.2-Update/Blocks-3.2.0.txt',
 		'properties': 'http://unicode.org/Public/3.2-Update/PropList-3.2.0.txt',
-		'derived-core-properties': 'http://unicode.org/Public/3.2-Update/DerivedCoreProperties-3.2.0.txt'
+		'derived-core-properties': 'http://unicode.org/Public/3.2-Update/DerivedCoreProperties-3.2.0.txt',
+		'case-folding': 'http://unicode.org/Public/3.2-Update/CaseFolding-3.2.0.txt'
 	},
 	{
 		'version': '4.0.0',
 		'main': 'http://unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt',
 		'scripts': 'http://unicode.org/Public/4.0-Update/Scripts-4.0.0.txt',
 		'blocks': 'http://unicode.org/Public/4.0-Update/Blocks-4.0.0.txt',
 		'properties': 'http://unicode.org/Public/4.0-Update/PropList-4.0.0.txt',
-		'derived-core-properties': 'http://unicode.org/Public/4.0-Update/DerivedCoreProperties-4.0.0.txt'
+		'derived-core-properties': 'http://unicode.org/Public/4.0-Update/DerivedCoreProperties-4.0.0.txt',
+		'case-folding': 'http://unicode.org/Public/4.0-Update/CaseFolding-4.0.0.txt'
 	},
 	{
 		'version': '4.0.1',
 		'main': 'http://unicode.org/Public/4.0-Update1/UnicodeData-4.0.1.txt',
 		'scripts': 'http://unicode.org/Public/4.0-Update1/Scripts-4.0.1.txt',
 		'blocks': 'http://unicode.org/Public/4.0-Update1/Blocks-4.0.1.txt',
 		'properties': 'http://unicode.org/Public/4.0-Update1/PropList-4.0.1.txt',
-		'derived-core-properties': 'http://unicode.org/Public/4.0-Update1/DerivedCoreProperties-4.0.1.txt'
+		'derived-core-properties': 'http://unicode.org/Public/4.0-Update1/DerivedCoreProperties-4.0.1.txt',
+		'case-folding': 'http://unicode.org/Public/4.0-Update1/CaseFolding-4.0.1.txt'
 	},
 	{
 		'version': '4.1.0',
@@ -73,6 +77,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/4.1.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/4.1.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/4.1.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/4.1.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/4.1.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -82,6 +87,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/5.0.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/5.0.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/5.0.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/5.0.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/5.0.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -90,7 +96,8 @@ var resources = [
 		'scripts': 'http://unicode.org/Public/5.1.0/ucd/Scripts.txt',
 		'blocks': 'http://unicode.org/Public/5.1.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/5.1.0/ucd/PropList.txt',
-		'derived-core-properties': 'http://unicode.org/Public/6.1.0/ucd/DerivedCoreProperties.txt',
+		'derived-core-properties': 'http://unicode.org/Public/5.1.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/5.1.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/5.1.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -100,6 +107,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/5.2.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/5.2.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/5.2.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/5.2.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/5.2.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -109,6 +117,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/6.0.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/6.0.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/6.0.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/6.0.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/6.0.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -118,6 +127,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/6.1.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/6.1.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/6.1.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/6.1.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/6.1.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -127,6 +137,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/6.2.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/6.2.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/6.2.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/6.2.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/6.2.0/ucd/BidiMirroring.txt'
 	},
 	{
@@ -136,6 +147,7 @@ var resources = [
 		'blocks': 'http://unicode.org/Public/6.3.0/ucd/Blocks.txt',
 		'properties': 'http://unicode.org/Public/6.3.0/ucd/PropList.txt',
 		'derived-core-properties': 'http://unicode.org/Public/6.3.0/ucd/DerivedCoreProperties.txt',
+		'case-folding': 'http://unicode.org/Public/6.3.0/ucd/CaseFolding.txt',
 		'bidi-mirroring': 'http://unicode.org/Public/6.3.0/ucd/BidiMirroring.txt',
 		'bidi-brackets': 'http://unicode.org/Public/6.3.0/ucd/BidiBrackets.txt',
 	}

index.js

@@ -1,6 +1,7 @@
 var utils = require('./scripts/utils.js');
 var parsers = require('./scripts/parse-blocks-scripts-properties.js');
 parsers.parseCategories = require('./scripts/parse-categories.js');
+parsers.parseCaseFolding = require('./scripts/parse-case-folding.js');
 var extend = utils.extend;
 var fs = require('fs');
 var jsesc = require('jsesc');
@@ -16,10 +17,7 @@ var compilePackage = template(fs.readFileSync(
 	path.resolve(templatePath, 'package.json'),
 	'utf-8')
 );
-var compileIndex = template(fs.readFileSync(
-	path.resolve(templatePath, 'index.js'),
-	'utf-8')
-);
+var compileIndex = template('module.exports=<%= data %>');
 
 var generateData = function(version) {
 	var dirMap = {};
@@ -56,6 +54,12 @@ var generateData = function(version) {
 		'map': parsers.parseDerivedCoreProperties(version),
 		'type': 'properties'
 	}));
+	console.log('Parsing Unicode v%s case folding…', version);
+	extend(dirMap, utils.writeFiles({
+		'version': version,
+		'map': parsers.parseCaseFolding(version),
+		'type': 'case-folding'
+	}));
 	console.log('Parsing Unicode v%s blocks…', version);
 	extend(dirMap, utils.writeFiles({
 		'version': version,
@@ -75,15 +79,15 @@ var generateData = function(version) {
 		'type': 'bidi-brackets'
 	}));
 	fs.writeFileSync(
-		path.resolve(__dirname, version, 'README.md'),
+		path.resolve(__dirname, 'output', 'unicode-' + version, 'README.md'),
 		compileReadMe({ 'version': version, 'dirs': dirMap })
 	);
 	fs.writeFileSync(
-		path.resolve(__dirname, version, 'index.js'),
+		path.resolve(__dirname, 'output', 'unicode-' + version, 'index.js'),
 		compileIndex({ 'version': version, 'data': jsesc(dirMap) })
 	);
 	fs.writeFileSync(
-		path.resolve(__dirname, version, 'package.json'),
+		path.resolve(__dirname, 'output', 'unicode-' + version, 'package.json'),
 		compilePackage({ 'version': version })
 	);
 	return dirMap;

package.json

@@ -50,13 +50,14 @@
 	"dependencies": {
 		"jsesc": "~0.4.3",
 		"lodash.template": "~2.4.1",
-		"mkdirp": "~0.4.0",
+		"mkdirp": "~0.5.0",
 		"punycode": "~1.2.4",
-		"regenerate": "~0.6.0",
-		"request": "~2.34.0",
-		"when": "~3.1.0"
+		"regenerate": "~0.6.2",
+		"request": "~2.36.0",
+		"string.fromcodepoint": "~0.2.0",
+		"when": "~3.2.2"
 	},
 	"devDependencies": {
-		"istanbul": "~0.2.7"
+		"istanbul": "~0.2.10"
 	}
 }

scripts/download.js

@@ -34,6 +34,7 @@ resources.forEach(function(resource) {
 		'blocks',
 		'properties',
 		'derived-core-properties',
+		'case-folding',
 		'bidi-mirroring',
 		'bidi-brackets'
 	].forEach(function(type) {

scripts/parse-case-folding.js

@@ -0,0 +1,26 @@
+var utils = require('./utils.js');
+
+var parseCaseFolding = function(version) {
+	var caseFoldingMap = {};
+	var source = utils.readDataFile(version, 'case-folding');
+	if (!source) {
+		return;
+	}
+	var lines = source.split('\n');
+	lines.forEach(function(line) {
+		if (!line || /^#/.test(line)) {
+			return;
+		}
+		var data = line.trim().split(';');
+		var codePoint = parseInt(data[0], 16);
+		var status = data[1].trim();
+		var mapping = parseInt(data[2], 16);
+		if (!caseFoldingMap[status]) {
+			caseFoldingMap[status] = {};
+		}
+		caseFoldingMap[status][codePoint] = mapping;
+	});
+	return caseFoldingMap;
+};
+
+module.exports = parseCaseFolding;

scripts/utils.js

@@ -2,8 +2,8 @@ var fs = require('fs');
 var path = require('path');
 var jsesc = require('jsesc');
 var regenerate = require('regenerate');
-var punycode = require('punycode');
 var mkdirp = require('mkdirp');
+require('string.fromcodepoint');
 
 var range = function(start, stop) {
 	// inclusive, e.g. `range(1, 3)` → `[1, 2, 3]`
@@ -38,13 +38,14 @@ var writeFiles = function(options) {
 		var type = typeof options.type == 'function'
 			? options.type(item)
 			: options.type;
+		var isCaseFolding = type == 'case-folding';
 		var isBidiProperty = type == 'bidi';
 		if (isBidiProperty) {
 			item = item.replace(/^Bidi_/, '');
 		}
 		var dir = path.resolve(
-			__dirname,
-			'..', version, type, item
+			__dirname, '..',
+			'output', 'unicode-' + version, type, item
 		);
 		if (
 			type == 'bidi' || type == 'bidi-mirroring' || type == 'bidi-brackets' ||
@@ -69,19 +70,30 @@ var writeFiles = function(options) {
 			path.resolve(dir, 'code-points.js'),
 			'module.exports=' + jsesc(codePoints)
 		);
-		fs.writeFileSync(
-			path.resolve(dir, 'regex.js'),
-			'module.exports=/' + regenerate(codePoints).toString() + '/'
-		);
+		if (!isCaseFolding) {
+			fs.writeFileSync(
+				path.resolve(dir, 'regex.js'),
+				'module.exports=/' + regenerate(codePoints).toString() + '/'
+			);
+		}
 		fs.writeFileSync(
 			path.resolve(dir, 'symbols.js'),
-			'module.exports=' + jsesc(codePoints.map(function(codePoint) {
-				return punycode.ucs2.encode([codePoint]);
-			}))
+			'module.exports=' + jsesc(
+				isCaseFolding ?
+					Object.keys(codePoints).reduce(function(result, current) {
+						result[String.fromCodePoint(current)] = String.fromCodePoint(codePoints[current]);
+						return result;
+					}, {}) :
+					codePoints.map(function(codePoint) {
+						return String.fromCodePoint(codePoint);
+					}))
 		);
 	});
 	Object.keys(auxMap).forEach(function(type) {
-		var dir = path.resolve(__dirname, '..', version, type);
+		var dir = path.resolve(
+			__dirname, '..',
+			'output', 'unicode-' + version, type
+		);
 		if (!hasKey(dirMap, type)) {
 			dirMap[type] = [];
 		}
@@ -126,8 +138,8 @@ var extend = function(destination, source) {
 
 var readDataFile = function(version, type) {
 	var sourceFile = path.resolve(
-		__dirname,
-		'..', 'data', version + '-' + type + '.txt'
+		__dirname, '..',
+		'data', version + '-' + type + '.txt'
 	);
 	if (!fs.existsSync(sourceFile)) {
 		return;

templates/README.md

@@ -1,6 +1,8 @@
 # Unicode v<%= version %> data
 
-JavaScript-compatible Unicode data for use in Node.js. Included: arrays of code points, arrays of symbols, and regular expressions for Unicode v<%= version %>’s categories, scripts, blocks, and properties.
+JavaScript-compatible Unicode data for use in Node.js. Included: arrays of code points, arrays of symbols, and regular expressions for Unicode v<%= version %>’s categories, scripts, blocks, and properties, as well as bidi mirroring and case folding data.
+
+Looking to build a JavaScript-compatible regular expression based on this data? [Use Regenerate.](http://mths.be/regenerate)
 
 The data files in this module are generated as part of the [node-unicode-data](http://mths.be/node-unicode-data) project.
 
@@ -47,11 +49,20 @@ require('unicode-<%= version %>/<%= type %>')[ codePoint ]; // lookup array
 <%
 	}
 	dirs[type].forEach(function(dir) {
+		if ('case-folding' == type) {
+%>
+require('unicode-<%= version %>/<%= type %>/<%= dir %>/code-points'); // lookup table with code point to code point mappings
+require('unicode-<%= version %>/<%= type %>/<%= dir %>/code-points')[ codePoint ]; // lookup table
+require('unicode-<%= version %>/<%= type %>/<%= dir %>/symbols'); // lookup table with symbol to symbol mappings
+require('unicode-<%= version %>/<%= type %>/<%= dir %>/symbols')[ symbol ]; // lookup table
+<%
+		} else {
 %>
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/code-points');
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/symbols');
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/regex');
 <%
+		}
 	});
 });
 %>```