Use Git submodules for third-party data.

Instead of downloading data using wget or relying on the Xorg compose
file being available on a Linux computer, we add these projects as Git
submodules (they all use Git) and simply refer to the source files.

Additionally, we embed those files as resources instead of copying
them around. The downside is that the user cannot hack them locally.
The upside is that the user cannot hack them locally.

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "src/3rdparty/xcompose"]
+	path = src/3rdparty/xcompose
+	url = ../fork-xcompose.git
+[submodule "src/3rdparty/unicode-translation"]
+	path = src/3rdparty/unicode-translation
+	url = ../unicode-translation.git
+[submodule "src/3rdparty/libx11"]
+	path = src/3rdparty/libx11
+	url = https://anongit.freedesktop.org/git/xorg/lib/libX11.git
+[submodule "src/3rdparty/x11proto"]
+	path = src/3rdparty/x11proto
+	url = https://anongit.freedesktop.org/git/xorg/proto/x11proto.git

src/Makefile

@@ -11,9 +11,7 @@ CONFIG = Release
 ISS = installer.iss
 PAS = installer.pas
 
-TXT = rules/Xorg.txt \
-      rules/Xcompose.txt \
-      rules/DefaultUserSequences.txt \
+TXT = rules/DefaultUserSequences.txt \
       rules/Emoji.txt \
       rules/WinCompose.txt
 

src/installer.iss

@@ -61,8 +61,6 @@ Source: "bin\{#CONFIG}\sr\*.dll"; DestDir: "{app}\sr"; Flags: ignoreversion
 Source: "bin\{#CONFIG}\sv\*.dll"; DestDir: "{app}\sv"; Flags: ignoreversion
 Source: "bin\{#CONFIG}\zh-CHS\*.dll"; DestDir: "{app}\zh-CHS"; Flags: ignoreversion
 Source: "bin\{#CONFIG}\zh-CHT\*.dll"; DestDir: "{app}\zh-CHT"; Flags: ignoreversion
-Source: "rules\Xorg.txt"; DestDir: "{app}\res"
-Source: "rules\Xcompose.txt"; DestDir: "{app}\res"
 Source: "rules\DefaultUserSequences.txt"; DestDir: "{app}\res"
 Source: "rules\Emoji.txt"; DestDir: "{app}\res"
 Source: "rules\WinCompose.txt"; DestDir: "{app}\res"
@@ -110,6 +108,9 @@ Filename: "{app}\{#EXE}"; Flags: nowait
 ; We used to be installed in c:\Program Files (x86)
 Type: filesandordirs; Name: "{pf32}\{#NAME}"
 ; Legacy stuff that we need to remove
+Type: files; Name: "{app}\rules\Xorg.txt"
+Type: files; Name: "{app}\rules\Xcompose.txt"
+Type: dirifempty; Name: "{app}\rules"
 Type: files; Name: "{app}\res\resources.dll"
 Type: files; Name: "{app}\res\wc.ico"
 Type: files; Name: "{app}\res\wca.ico"

src/sequences/Key.cs

@@ -64,7 +64,7 @@ public partial class Key
     private static Dictionary<string, string> ReadXorgKeySyms()
     {
         Dictionary<string, string> ret = new Dictionary<string, string>();
-        using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream("WinCompose.res.keysymdef.h"))
+        using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream("3rdparty.keysymdef.h"))
         using (StreamReader reader = new StreamReader(s))
         {
             Regex r = new Regex(@"^#define XK_([^ ]*).* U\+([A-Za-z0-9]+)");

src/settings/Settings.cs

@@ -255,8 +255,9 @@ public static void LoadSequences()
             m_sequences = new SequenceTree();
             m_sequence_count = 0;
 
-            LoadSequenceFile(Path.Combine(GetDataDir(), "Xorg.txt"));
-            LoadSequenceFile(Path.Combine(GetDataDir(), "XCompose.txt"));
+            LoadSequenceResource("3rdparty.xorg.rules");
+            LoadSequenceResource("3rdparty.xcompose.rules");
+
             LoadSequenceFile(Path.Combine(GetDataDir(), "Emoji.txt"));
             LoadSequenceFile(Path.Combine(GetDataDir(), "WinCompose.txt"));
 
@@ -356,9 +357,11 @@ private static void LoadSequenceFile(string path)
         {
             try
             {
-                foreach (string line in File.ReadAllLines(path))
-                    LoadSequenceString(line);
-                Log.Debug("Loaded rule file {0}", path);
+                using (StreamReader s = new StreamReader(path))
+                {
+                    Log.Debug("Loaded rule file {0}", path);
+                    LoadSequenceStream(s);
+                }
             }
             catch (FileNotFoundException)
             {
@@ -367,13 +370,36 @@ private static void LoadSequenceFile(string path)
             catch (Exception) { }
         }
 
+        private static void LoadSequenceResource(string resource)
+        {
+            using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream(resource))
+            using (StreamReader sr = new StreamReader(s))
+            {
+                Log.Debug("Loaded rule resource {0}", resource);
+                LoadSequenceStream(sr);
+            }
+        }
+
+        private static void LoadSequenceStream(StreamReader s)
+        {
+            Regex match_comment = new Regex(@"/\*([^*]|\*[^/])*\*/");
+
+            /* Read file and remove all C comments */
+            string buffer = s.ReadToEnd();
+            buffer = match_comment.Replace(buffer, "");
+
+            /* Parse all lines */
+            foreach (string line in buffer.Split('\r', '\n'))
+                ParseRule(line);
+        }
+
         private static Regex m_r0 = new Regex(@"^\s*include\s*""([^""]*)""");
         private static Regex m_r1 = new Regex(@"^\s*<Multi_key>\s*([^:]*):[^""]*""(([^""]|\\"")*)""[^#]*#?\s*(.*)");
             //                                                    ^^^^^^^         ^^^^^^^^^^^^^^^            ^^^^
             //                                                     keys               result                 desc
         private static Regex m_r2 = new Regex(@"[\s<>]+");
 
-        private static void LoadSequenceString(string line)
+        private static void ParseRule(string line)
         {
             // If this is an include directive, use LoadSequenceFile() again
             Match m0 = m_r0.Match(line);
@@ -442,9 +468,9 @@ private static void LoadSequenceString(string line)
             int utf32 = StringToCodepoint(result);
             if (utf32 >= 0)
             {
-                string key = String.Format("U{0:X04}", utf32);
+                string key = string.Format("U{0:X04}", utf32);
                 string alt_desc = unicode.Char.ResourceManager.GetString(key);
-                if (alt_desc != null && alt_desc.Length > 0)
+                if (!string.IsNullOrEmpty(alt_desc))
                     description = alt_desc;
             }
 
@@ -458,7 +484,7 @@ private static int StringToCodepoint(string s)
                 return (int)s[0];
 
             if (s.Length == 2 && s[0] >= 0xd800 && s[0] <= 0xdbff)
-                return Char.ConvertToUtf32(s[0], s[1]);
+                return char.ConvertToUtf32(s[0], s[1]);
 
             return -1;
         }

src/update-data.sh

@@ -2,30 +2,14 @@
 
 set -e
 
-STEPS=6
-CACHE=unicode/cache
-mkdir -p ${CACHE}
-
-#
-# Copy and transform system files
-#
-
-echo "[1/${STEPS}] Copy system files…"
-
-if [ -f /usr/share/X11/locale/en_US.UTF-8/Compose ]; then
-    cat -s /usr/share/X11/locale/en_US.UTF-8/Compose > rules/Xorg.txt
-fi
-
-if [ -f /usr/include/X11/keysymdef.h ]; then
-    cat -s /usr/include/X11/keysymdef.h > res/keysymdef.h
-fi
+STEPS=5
 
 #
 # Rebuild po/wincompose.pot from our master translation file Text.resx
 # then update all .po files
 #
 
-echo "[2/${STEPS}] Rebuild potfiles…"
+echo "[1/${STEPS}] Rebuild potfiles…"
 DEST=po/wincompose.pot
 # Update POT-Creation-Date with: date +'%Y-%m-%d %R%z'
 cat > ${DEST} << EOF
@@ -106,7 +90,7 @@ po2res()
     esac
 }
 
-echo "[3/${STEPS}] Rebuild resx files…"
+echo "[2/${STEPS}] Rebuild resx files…"
 for POFILE in po/*.po; do
     polang=$(basename ${POFILE} .po)
     reslang=$(po2res $polang)
@@ -148,16 +132,11 @@ done
 # and create .resx translation files for our project
 #
 
-echo "[4/${STEPS}] Rebuild Unicode translation files…"
-INDEX=https://github.com/samhocevar/unicode-translation/tree/master/po
-BASE=https://raw.github.com/samhocevar/unicode-translation/master/po/
-PO=$(wget -qO- $INDEX | tr '<>' '\n' | sed -ne 's/^\(..\)[.]po$/\1/p')
-for polang in $PO; do
-    printf "${polang}... "
+echo "[3/${STEPS}] Rebuild Unicode translation files…"
+for POFILE in 3rdparty/unicode-translation/po/*.po; do
+    polang=$(basename ${POFILE} .po)
     reslang=$(po2res $polang)
-    SRC=${CACHE}/${polang}.po
-    # Get latest translation if new
-    (cd ${CACHE} && wget -q -N ${BASE}/${polang}.po)
+    printf "${polang}... "
 
     # Parse data and put it in the Char.*.resx and Block.*.resx files
     for FILE in Char Block; do
@@ -173,7 +152,7 @@ for polang in $PO; do
         esac
         DEST=unicode/${FILE}.${reslang}.resx
         sed -e '/^  <data/,$d' < unicode/${FILE}.resx > ${DEST}
-        if uname | grep -qi mingw; then unix2dos; else cat; fi < ${SRC} \
+        if uname | grep -qi mingw; then unix2dos; else cat; fi < ${POFILE} \
           | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' \
           | awk 'function f() {
                      if (c && msgstr) {
@@ -196,7 +175,7 @@ echo "done."
 # Check some wincompose.csproj consistency
 #
 
-echo "[5/${STEPS}] Check consistency…"
+echo "[4/${STEPS}] Check consistency…"
 for x in unicode/*.*.resx i18n/*.*.resx; do
     reslang="$(echo $x | cut -f2 -d.)"
     if ! grep -q '"'$(echo $x | tr / .)'"' wincompose.csproj; then
@@ -223,7 +202,7 @@ fi
 # Build translator list
 #
 
-echo "[6/${STEPS}] Update contributor list…"
+echo "[5/${STEPS}] Update contributor list…"
 printf '' > res/.contributors.html
 cat >> res/.contributors.html << EOF
 <html>

src/wincompose.csproj

@@ -993,7 +993,15 @@
   <ItemGroup>
     <None Include="app.config" />
     <EmbeddedResource Include="build.config" />
-    <EmbeddedResource Include="res\keysymdef.h" />
+    <EmbeddedResource Include="3rdparty\x11proto\keysymdef.h">
+        <LogicalName>3rdparty.keysymdef.h</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="3rdparty\libx11\nls\en_US.UTF-8\Compose.pre">
+        <LogicalName>3rdparty.xorg.rules</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="3rdparty\xcompose\dotXCompose">
+        <LogicalName>3rdparty.xcompose.rules</LogicalName>
+    </EmbeddedResource>
   </ItemGroup>
   <ItemGroup>
     <Resource Include="res\contributors.html" />
@@ -1012,8 +1020,6 @@
     <Resource Include="rules\DefaultUserSequences.txt" />
     <Resource Include="rules\Emoji.txt" />
     <Resource Include="rules\WinCompose.txt" />
-    <Resource Include="rules\Xcompose.txt" />
-    <Resource Include="rules\Xorg.txt" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it.