Permalink
Browse files

[Cursor] Following a so-simple-I-feel-stupid suggestion from pmurias+…

…+, we now keep our own copy of part of the Unicode property database. As a fringe benefit, it's much more similar to the internal form and can be loaded in 0.1s less time.

git-svn-id: http://svn.pugscode.org/pugs@31356 c213334d-75ef-0310-aa23-eaa082d1ae64
  • Loading branch information...
1 parent 18b0b4b commit 78f42784e33898755a13efe5616b3988faa74de9 sorear committed Jun 18, 2010
Showing with 62 additions and 15 deletions.
  1. +25 −14 CursorBase.pmc
  2. +1 −1 README
  3. +36 −0 gen-unicode-table.pl
  4. BIN uniprops
View
@@ -1820,22 +1820,33 @@ sub bless { CORE::bless $_[1], $_[0]->WHAT }
# Steal data from Perl5's Unicode maps
my %unicode_map_cache;
-$unicode_map_cache{ALL} = [scalar("\377" x 128) x 1088, "ALL"] ;
-sub _get_unicode_map {
- my $propname = shift;
- $unicode_map_cache{$propname} //= do {
- my @top = (("") x 1088, $propname);
- for my $l (split("\n", (do "unicore/lib/$propname.pl"))) {
- my ($from, $to) = split("\t", $l);
- $from = hex $from;
- $to = hex $to || $from;
-
- for (my $x = $from; $x <= $to; $x++) {
- vec($top[$x >> 10], $x & 1023, 1) = 1;
+BEGIN {
+ $unicode_map_cache{ALL} = [scalar("\377" x 128) x 1088, "ALL"] ;
+ open MAP, "uniprops" or die "cannot open unicode maps: $!";
+ binmode MAP;
+ while (defined (my $c = getc MAP)) {
+ my $name = "";
+ my $used;
+ my $tile;
+ read MAP, $name, ord($c);
+ read MAP, $used, 136;
+
+ $unicode_map_cache{$name} = [];
+ $unicode_map_cache{$name}[1088] = $name;
+
+ for (my $i = 0; $i < 1088; $i++) {
+ if (vec($used, $i, 1)) {
+ read MAP, $tile, 128;
+ $unicode_map_cache{$name}[$i] = $tile;
}
}
- \@top;
- };
+ }
+ close MAP or die "cannot close unicode maps: $!";
+}
+sub _get_unicode_map {
+ my $propname = shift;
+ $unicode_map_cache{$propname} //
+ die "Map $propname not found. Edit gen-unicode-table.pl and rerun.";
}
# This is the fast path handling for JIT DFA lexer generation (although it gets
View
2 README
@@ -14,7 +14,7 @@ minor tools used in the development of STD.pm6 and viv.
You'll need the following Perl bits to run stuff:
-* Perl 5.12.0 or later in /usr/local/bin/
+* Perl 5.10.0 or later in /usr/local/bin/
(using current perl with eg `sudo ln -s /usr/bin/perl /usr/local/bin`)
* YAML::XS (eg sudo apt-get install libyaml-libyaml-perl)
View
@@ -0,0 +1,36 @@
+#! /usr/local/bin/perl
+use 5.012; # unicore format is unstable
+use strict;
+use warnings;
+use autodie;
+
+my @tables = (qw|Gc/N Gc/L Perl/Blank Space/Y Perl/VertSpac|);
+
+open DUMP, ">", "uniprops";
+binmode DUMP;
+
+for my $propname (@tables) {
+ my @top = (("\0" x 128) x 1088);
+ my $used = "\0" x 136;
+ for my $l (split("\n", (do "unicore/lib/$propname.pl"))) {
+ my ($from, $to) = split("\t", $l);
+ $from = hex $from;
+ $to = hex $to || $from;
+
+ for (my $x = $from; $x <= $to; $x++) {
+ vec($top[$x >> 10], $x & 1023, 1) = 1;
+ vec($used, $x >> 10, 1) = 1;
+ }
+ }
+
+ print DUMP chr(length($propname));
+ print DUMP $propname;
+ print DUMP $used;
+ for (my $i = 0; $i < 1088; $i++) {
+ if (vec($used, $i, 1)) {
+ print DUMP $top[$i];
+ }
+ }
+}
+
+close DUMP;
View
BIN uniprops
Binary file not shown.

0 comments on commit 78f4278

Please sign in to comment.