Support DDTP translations

source/backends/debian/debpkgindex.d

@@ -33,7 +33,7 @@ import backends.debian.tagfile;
 import backends.debian.debpkg;
 import backends.debian.debutils;
 import config;
-import utils : escapeXml, isRemote;
+import utils : escapeXml, getFileContents, isRemote;
 
 
 class DebianPackageIndex : PackageIndex
@@ -63,65 +63,109 @@ public:
         indexChanged = null;
     }
 
-    private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
+    private immutable(string[]) findTranslations (const string suite, const string section)
     {
-        immutable enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s");
-        string enDescFname;
+        import std.regex : ctRegex, matchFirst;
+
+        immutable inRelease = buildPath (rootDir, "dists", suite, "InRelease");
+        auto regex = ctRegex!(r"Translation-(\w+)$");
+        bool[string] ret;
 
         try {
-            enDescFname = downloadIfNecessary (rootDir, tmpDir, enDescPath);
-        } catch (Exception e) {
-            logDebug ("No long descriptions for %s/%s: %s", suite, section, e.msg);
-            return;
+            const inReleaseContents = getFileContents (inRelease);
+
+            foreach (const ref entry; inReleaseContents) {
+                auto match = entry.matchFirst(regex);
+
+                if (match.empty)
+                    continue;
+
+                ret[match[1]] = true;
+            }
+        } catch (Exception ex) {
+            logWarning ("Couldn't download %s, will assume 'en' is available.", inRelease);
+            return ["en"];
         }
 
-        auto tagf = new TagFile ();
-        tagf.open (enDescFname);
+        return cast (immutable) ret.keys;
+    }
 
-        logDebug ("Opened: %s", enDescFname);
-        do {
-            auto pkgname = tagf.readField ("Package");
-            auto rawDesc  = tagf.readField ("Description-en");
-            if (!pkgname)
-                continue;
-            if (!rawDesc)
-                continue;
+    private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
+    {
+        immutable langs = findTranslations (suite, section);
+
+        foreach (const ref lang; langs) {
+            string fname;
+
+            immutable fullPath = buildPath ("dists",
+                                            suite,
+                                            section,
+                                            "i18n",
+                                            /* here we explicitly substitute a
+                                             * "%s", because
+                                             * downloadIfNecessary will put the
+                                             * file extension there */
+                                            "Translation-%s.%s".format(lang, "%s"));
+
+            try {
+                fname = downloadIfNecessary (rootDir, tmpDir, fullPath);
+            } catch (Exception ex) {
+                logDebug ("No long descriptions for %s/%s", suite, section);
+                return;
+            }
 
-            auto pkgP = (pkgname in pkgs);
-            if (pkgP is null)
-                continue;
+            auto tagf = new TagFile ();
+            tagf.open (fname);
 
-            auto split = rawDesc.split ("\n");
-            if (split.length < 2)
-                continue;
+            logDebug ("Opened: %s", fname);
+            do {
+                auto pkgname = tagf.readField ("Package");
+                auto rawDesc  = tagf.readField ("Description-%s".format (lang));
+                if (!pkgname)
+                    continue;
+                if (!rawDesc)
+                    continue;
 
-            // NOTE: .remove() removes the element, but does not alter the length of the array. Bug?
-            // (this is why we slice the array here)
-            split = split[1..$];
-
-            // TODO: We actually need a Markdown-ish parser here if we want to support
-            // listings in package descriptions properly.
-            auto description = appender!string;
-            description ~= "<p>";
-            bool first = true;
-            foreach (l; split) {
-                if (l.strip () == ".") {
-                    description ~= "</p>\n<p>";
-                    first = true;
+                auto pkgP = (pkgname in pkgs);
+                if (pkgP is null)
                     continue;
-                }
 
-                if (first)
-                    first = false;
-                else
-                    description ~= " ";
+                auto split = rawDesc.split ("\n");
+                if (split.length < 2)
+                    continue;
 
-                description ~= escapeXml (l);
-            }
-            description ~= "</p>";
+                // NOTE: .remove() removes the element, but does not alter the
+                // length of the array. Bug?  (this is why we slice the array
+                // here)
+                split = split[1..$];
+
+                // TODO: We actually need a Markdown-ish parser here if we want
+                // to support listings in package descriptions properly.
+                auto description = appender!string;
+                description ~= "<p>";
+                bool first = true;
+                foreach (l; split) {
+                    if (l.strip () == ".") {
+                        description ~= "</p>\n<p>";
+                        first = true;
+                        continue;
+                    }
+
+                    if (first)
+                        first = false;
+                    else
+                        description ~= " ";
+
+                    description ~= escapeXml (l);
+                }
+                description ~= "</p>";
 
-            (*pkgP).setDescription (description.data, "C");
-        } while (tagf.nextSection ());
+                if (lang == "en")
+                    (*pkgP).setDescription (description.data, "C");
+
+                (*pkgP).setDescription (description.data, lang);
+            } while (tagf.nextSection ());
+        }
     }
 
     private string getIndexFile (string suite, string section, string arch)
@@ -219,3 +263,16 @@ public:
         return false;
     }
 }
+
+unittest {
+    import std.algorithm.sorting : sort;
+
+    writeln ("TEST: ", "DebianPackageIndex");
+
+    auto pi = new DebianPackageIndex (buildPath (getcwd (), "test", "samples", "debian"));
+    assert (sort(pi.findTranslations ("sid", "main").dup) ==
+            sort(["en", "ca", "cs", "da", "de", "de_DE", "el", "eo", "es",
+                   "eu", "fi", "fr", "hr", "hu", "id", "it", "ja", "km", "ko",
+                   "ml", "nb", "nl", "pl", "pt", "pt_BR", "ro", "ru", "sk",
+                   "sr", "sv", "tr", "uk", "vi", "zh", "zh_CN", "zh_TW"]));
+}

source/result.d

@@ -223,11 +223,15 @@ public:
         // inject package descriptions, if needed
         foreach (cpt; cpts.byValue ()) {
             if (cpt.getKind () == ComponentKind.DESKTOP_APP) {
-                cpt.setActiveLocale ("C");
-                if (cpt.getDescription ().empty) {
-                    auto descP = "C" in pkg.description;
-                    if (descP !is null) {
-                        cpt.setDescription (*descP, "C");
+                auto flags = cpt.getValueFlags;
+                cpt.setValueFlags (flags | AsValueFlags.NO_TRANSLATION_FALLBACK);
+                scope (exit) cpt.setActiveLocale ("C");
+
+                foreach (const string lang, const string desc; pkg.description) {
+                    cpt.setActiveLocale (lang);
+
+                    if (cpt.getDescription ().empty) {
+                        cpt.setDescription (desc, lang);
                         addHint (cpt.getId (), "description-from-package");
                     }
                 }

source/utils.d

@@ -321,63 +321,37 @@ bool isRemote (const string uri)
     return (!match.empty);
 }
 
-/**
- * Download `url` to `dest`.
- *
- * Params:
- *      url = The URL to download.
- *      dest = The location for the downloaded file.
- *      retryCount = Number of times to retry on timeout.
- */
-void downloadFile (const string url, const string dest, const uint retryCount = 5)
+private void download (const string url, ref File dest, const uint retryCount = 5)
 in
 {
-    assert (isRemote (url));
-}
-out
-{
-    assert (std.file.exists (dest));
+    assert (url.isRemote);
 }
 body
 {
     import core.time;
 
-    import std.file;
-    import std.net.curl;
-    import std.path;
-
-
-    if (dest.exists) {
-        logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
-        return;
-    }
+    import std.net.curl : CurlTimeoutException, HTTP, FTP;
 
     ulong onReceiveCb (File f, ubyte[] data)
     {
         f.rawWrite (data);
         return data.length;
     }
 
-    mkdirRecurse (dest.dirName);
-
     /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
     logDebug ("Downloading %s", url);
     try {
-        auto f = File (dest, "wb");
-        scope(exit) f.close();
-        scope(failure) remove(dest);
-
         if (url.startsWith ("http")) {
             auto downloader = HTTP (url);
             downloader.connectTimeout = dur!"seconds" (30);
             downloader.dataTimeout = dur!"seconds" (30);
-            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.onReceive = (data) => onReceiveCb (dest, data);
             downloader.perform();
         } else {
             auto downloader = FTP (url);
             downloader.connectTimeout = dur!"seconds" (30);
             downloader.dataTimeout = dur!"seconds" (30);
-            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.onReceive = (data) => onReceiveCb (dest, data);
             downloader.perform();
         }
         logDebug ("Downloaded %s", url);
@@ -387,13 +361,84 @@ body
                       url,
                       retryCount,
                       retryCount > 1 ? "times" : "time");
-            downloadFile (url, dest, retryCount - 1);
+            download (url, dest, retryCount - 1);
         } else {
             throw e;
         }
     }
 }
 
+/**
+ * Download or open `path` and return it as a string array.
+ *
+ * Params:
+ *      path = The path to access.
+ *
+ * Returns: The data if successful.
+ */
+string[] getFileContents (const string path, const uint retryCount = 5)
+{
+    import core.stdc.stdlib : free;
+    import core.sys.linux.stdio : fclose, open_memstream;
+
+    char * ptr = null;
+    scope (exit) free (ptr);
+
+    size_t sz = 0;
+
+    auto f = open_memstream (&ptr, &sz);
+    scope (exit) fclose (f);
+
+    auto file = File.wrapFile (f);
+
+    if (path.isRemote) {
+        download (path, file, retryCount);
+    } else {
+        if (!std.file.exists (path))
+            throw new Exception ("No such file '%s'", path);
+
+        return std.file.readText (path).splitLines;
+    }
+
+    return to!string (ptr.fromStringz).splitLines;
+}
+
+/**
+ * Download `url` to `dest`.
+ *
+ * Params:
+ *      url = The URL to download.
+ *      dest = The location for the downloaded file.
+ *      retryCount = Number of times to retry on timeout.
+ */
+void downloadFile (const string url, const string dest, const uint retryCount = 5)
+in
+{
+    assert (url.isRemote);
+}
+out
+{
+    assert (std.file.exists (dest));
+}
+body
+{
+    import std.file;
+    import std.path;
+
+    if (dest.exists) {
+        logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
+        return;
+    }
+
+    mkdirRecurse (dest.dirName);
+
+    auto f = File (dest, "wb");
+    scope (exit) f.close();
+    scope (failure) remove(dest);
+
+    download (url, f, retryCount);
+}
+
 unittest
 {
     writeln ("TEST: ", "GCID");