debian: Support downloading from a mirror in addition to having it locally

source/backends/debian/debpkg.d

@@ -21,13 +21,15 @@ module ag.backend.debian.debpkg;
 
 import std.stdio;
 import std.string;
+import std.path;
 import std.array : empty, appender;
 import std.file : rmdirRecurse, mkdirRecurse;
 static import std.file;
 import ag.config;
 import ag.archive;
 import ag.backend.intf;
 import ag.logging;
+import ag.utils : isRemote, downloadFile;
 
 
 class DebPackage : Package
@@ -56,7 +58,16 @@ public:
     @property override const(string[string]) description () const { return desc; }
 
     override
-    @property string filename () const { return debFname; }
+    @property string filename () const {
+        if (debFname.isRemote) {
+            immutable path = buildPath (tmpDir, debFname.baseName);
+
+            downloadFile (debFname, path);
+
+            return path;
+        }
+        return debFname;
+    }
     @property void   filename (string fname) { debFname = fname; }
 
     override
@@ -65,8 +76,6 @@ public:
 
     this (string pname, string pver, string parch)
     {
-        import std.path;
-
         pkgname = pname;
         pkgver = pver;
         pkgarch = parch;
@@ -94,7 +103,6 @@ public:
         auto pa = new ArchiveDecompressor ();
         if (!dataArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             pa.open (this.filename);
@@ -119,7 +127,6 @@ public:
         auto ca = new ArchiveDecompressor ();
         if (!controlArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             ca.open (this.filename);

source/backends/debian/debpkgindex.d

@@ -30,7 +30,9 @@ import ag.logging;
 import ag.backend.intf;
 import ag.backend.debian.tagfile;
 import ag.backend.debian.debpkg;
-import ag.utils : escapeXml;
+import ag.backend.debian.utils;
+import ag.config;
+import ag.utils : escapeXml, isRemote;
 
 
 class DebianPackageIndex : PackageIndex
@@ -40,14 +42,18 @@ private:
     string rootDir;
     Package[][string] pkgCache;
     bool[string] indexChanged;
+    string tmpDir;
 
 public:
 
     this (string dir)
     {
         this.rootDir = dir;
-        if (!std.file.exists (dir))
+        if (!dir.isRemote && !std.file.exists (dir))
             throw new Exception ("Directory '%s' does not exist.".format (dir));
+
+        auto conf = Config.get ();
+        tmpDir = buildPath (conf.getTmpDir (), dir.baseName);
     }
 
     void release ()
@@ -58,8 +64,12 @@ public:
 
     private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
     {
-        auto enDescFname = buildPath (rootDir, "dists", suite, section, "i18n", "Translation-en.bz2");
-        if (!std.file.exists (enDescFname)) {
+        immutable enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s");
+        string enDescFname;
+
+        try {
+            enDescFname = downloadIfNecessary (rootDir, tmpDir, enDescPath);
+        } catch (Exception ex) {
             logDebug ("No long descriptions for %s/%s", suite, section);
             return;
         }
@@ -119,11 +129,9 @@ public:
 
     private string getIndexFile (string suite, string section, string arch)
     {
-        immutable binDistsPath = buildPath (rootDir, "dists", suite, section, "binary-%s".format (arch));
-        auto indexFname = buildPath (binDistsPath, "Packages.gz");
-        if (!std.file.exists (indexFname))
-            indexFname = buildPath (binDistsPath, "Packages.xz");
-        return indexFname;
+        immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch));
+
+        return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s"));
     }
 
     private DebPackage[] loadPackages (string suite, string section, string arch)

source/backends/debian/utils.d

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2016 Canonical Ltd
+ * Author(s): Iain Lane <iain@orangesquash.org.uk>
+ *
+ * Licensed under the GNU Lesser General Public License Version 3
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the license, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this software.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+module ag.backend.debian.utils;
+
+import std.string;
+
+import ag.logging;
+import ag.utils : downloadFile, isRemote;
+
+/**
+ * If prefix is remote, download the first of (prefix + suffix).{xz,bz2,gz},
+ * otherwise check if any of (prefix + suffix).{xz,bz2,gz} exists.
+ *
+ * Returns: Path to the file, which is guaranteed to exist.
+ *
+ * Params:
+ *      prefix = First part of the address, i.e.
+ *               "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/"
+ *      destPrefix = If the file is remote, the directory to save it under,
+ *                   which is created if necessary.
+ *      suffix = the rest of the address, so that (prefix +
+ *               suffix).format({xz,bz2,gz}) is a full path or URL, i.e.
+ *               "dists/unstable/main/binary-i386/Packages.%s". The suffix must
+ *               contain exactly one "%s"; this function is only suitable for
+ *               finding `.xz`, `.bz2` and `.gz` files.
+ */
+immutable (string) downloadIfNecessary (const string prefix,
+                                        const string destPrefix,
+                                        const string suffix)
+{
+    import std.net.curl;
+    import std.path;
+
+    immutable exts = ["xz", "bz2", "gz"];
+    foreach (ref ext; exts) {
+        immutable fileName = format (buildPath (prefix, suffix), ext);
+        immutable destFileName = format (buildPath (destPrefix, suffix), ext);
+
+        if (fileName.isRemote) {
+            try {
+                /* This should use download(), but that doesn't throw errors */
+                downloadFile (fileName, destFileName);
+
+                return destFileName;
+            } catch (CurlException ex) {
+                logDebug ("Couldn't download: %s", ex.msg);
+            }
+        } else {
+            if (std.file.exists (fileName))
+                return fileName;
+        }
+    }
+
+    /* all extensions failed, so we failed */
+    throw new Exception (format ("Couldn't obtain any file matching %s",
+                         buildPath (prefix, suffix)));
+}

source/utils.d

@@ -19,7 +19,9 @@
 
 module ag.utils;
 
-import std.stdio : writeln;
+import ag.logging;
+
+import std.stdio : File, write, writeln;
 import std.string;
 import std.ascii : letters, digits;
 import std.conv : to;
@@ -307,6 +309,92 @@ ubyte[] stringArrayToByteArray (string[] strArray) pure @trusted
     return res.data;
 }
 
+@safe
+bool isRemote (const string uri)
+{
+    import std.regex;
+
+    auto uriregex = ctRegex!(`^(https?|ftps?)://`);
+
+    auto match = matchFirst(uri, uriregex);
+
+    return (!match.empty);
+}
+
+private ulong onReceiveCb (File f, ubyte[] data)
+{
+    f.rawWrite (data);
+
+    return data.length;
+}
+
+/**
+ * Download `url` to `dest`.
+ *
+ * Params:
+ *      url = The URL to download.
+ *      dest = The location for the downloaded file.
+ *      retryCount = Number of times to retry on timeout.
+ */
+void downloadFile (const string url, const string dest, const uint retryCount = 5)
+in
+{
+    assert (isRemote (url));
+}
+out
+{
+    assert (std.file.exists (dest));
+}
+body
+{
+    import core.time;
+
+    import std.file;
+    import std.net.curl;
+    import std.path;
+
+
+    if (dest.exists) {
+        logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
+        return;
+    }
+
+    mkdirRecurse (dest.dirName);
+
+    /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
+    logDebug ("Downloading %s", url);
+    try {
+        auto f = File (dest, "wb");
+        scope(exit) f.close();
+        scope(failure) remove(dest);
+
+        if (url.startsWith ("http")) {
+            auto downloader = HTTP (url);
+            downloader.connectTimeout = dur!"seconds" (30);
+            downloader.dataTimeout = dur!"seconds" (30);
+            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.perform();
+        } else {
+            auto downloader = FTP (url);
+            downloader.connectTimeout = dur!"seconds" (30);
+            downloader.dataTimeout = dur!"seconds" (30);
+            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.perform();
+        }
+        logDebug ("Downloaded %s", url);
+    } catch (CurlTimeoutException e) {
+        if (retryCount > 0) {
+            logDebug ("Failed to download %s, will retry %d more %s",
+                      url,
+                      retryCount,
+                      retryCount > 1 ? "times" : "time");
+            downloadFile (url, dest, retryCount - 1);
+        } else {
+            throw e;
+        }
+    }
+}
+
 unittest
 {
     writeln ("TEST: ", "GCID");
@@ -327,4 +415,7 @@ unittest
     assert (ImageSize (48) < ImageSize (64));
 
     assert (stringArrayToByteArray (["A", "b", "C", "ö", "8"]) == [65, 98, 67, 195, 182, 56]);
+
+    assert (isRemote ("http://test.com"));
+    assert (!isRemote ("/srv/"));
 }