Support downloading files from a mirror, instead of having them local

source/backends/debian/debpkg.d

@@ -21,12 +21,14 @@ module ag.backend.debian.debpkg;
 
 import std.stdio;
 import std.string;
+import std.path;
 import std.array : empty, appender;
 import std.file : rmdirRecurse;
 import ag.config;
 import ag.archive;
 import ag.backend.intf;
 import ag.logging;
+import ag.utils : isRemote, downloadFile;
 
 
 class DebPackage : Package
@@ -52,15 +54,23 @@ public:
     @property string ver () const { return pkgver; }
     @property string arch () const { return pkgarch; }
     @property const(string[string]) description () const { return desc; }
-    @property string filename () const { return debFname; }
+    @property string filename () const {
+        if (debFname.isRemote) {
+            immutable auto path = buildPath (tmpDir, debFname.baseName);
+
+            /* XXX: Retry? */
+            downloadFile (debFname, path);
+
+            return path;
+        }
+        return debFname;
+    }
     @property void filename (string fname) { debFname = fname; }
     @property string maintainer () const { return pkgmaintainer; }
     @property void maintainer (string maint) { pkgmaintainer = maint; }
 
     this (string pname, string pver, string parch)
     {
-        import std.path;
-
         pkgname = pname;
         pkgver = pver;
         pkgarch = parch;
@@ -101,7 +111,6 @@ public:
         auto pa = new ArchiveDecompressor ();
         if (!dataArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             pa.open (this.filename);
@@ -126,7 +135,6 @@ public:
         auto ca = new ArchiveDecompressor ();
         if (!controlArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             ca.open (this.filename);

source/backends/debian/debpkgindex.d

@@ -28,6 +28,8 @@ import ag.logging;
 import ag.backend.intf;
 import ag.backend.debian.tagfile;
 import ag.backend.debian.debpkg;
+import ag.config;
+import ag.utils : isRemote, downloadFile;
 
 
 class DebianPackageIndex : PackageIndex
@@ -37,14 +39,18 @@ private:
     string rootDir;
     Package[][string] pkgCache;
     bool[string] indexChanged;
+    string tmpDir;
 
 public:
 
     this (string dir)
     {
         this.rootDir = dir;
-        if (!std.file.exists (dir))
+        if (!dir.isRemote && !std.file.exists (dir))
             throw new Exception ("Directory '%s' does not exist.".format (dir));
+
+        auto conf = Config.get ();
+        tmpDir = buildPath (conf.getTmpDir (), dir.baseName);
     }
 
     void release ()
@@ -55,7 +61,19 @@ public:
 
     private void loadPackageLongDescs (Package[string] pkgs, string suite, string section)
     {
-        auto enDescFname = buildPath (rootDir, "dists", suite, section, "i18n", "Translation-en.bz2");
+        immutable auto enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.xz");
+
+        string enDescFname;
+
+        if (rootDir.isRemote) {
+            enDescFname = buildPath (tmpDir, enDescPath);
+            immutable auto uri = buildPath (rootDir, enDescPath);
+
+            downloadFile (uri, enDescFname);
+        } else {
+            enDescFname = buildPath (rootDir, enDescPath);
+        }
+
         if (!std.file.exists (enDescFname)) {
             logDebug ("No long descriptions for %s/%s", suite, section);
             return;
@@ -116,10 +134,38 @@ public:
 
     private string getIndexFile (string suite, string section, string arch)
     {
-        auto binDistsPath = buildPath (rootDir, "dists", suite, section, "binary-%s".format (arch));
-        auto indexFname = buildPath (binDistsPath, "Packages.gz");
-        if (!std.file.exists (indexFname))
-            indexFname = buildPath (binDistsPath, "Packages.xz");
+        auto path = buildPath ("dists", suite, section, "binary-%s".format (arch));
+        auto binDistsPath = buildPath (rootDir, path);
+
+        string indexFname;
+
+        if (rootDir.isRemote) {
+            import std.file;
+            import std.net.curl : CurlException;
+
+            foreach (string ext; ["xz", "gz"]) {
+                try {
+                    indexFname = buildPath (tmpDir, path, format ("Packages.%s", ext));
+                    immutable auto uri = buildPath (binDistsPath, format ("Packages.%s", ext));
+
+                    /* This should use download(), but that doesn't throw errors */
+                    downloadFile (uri, indexFname);
+
+                    break;
+                } catch (CurlException ex) {
+                    logDebug ("Couldn't download: %s", ex.msg);
+                    indexFname = null;
+                }
+            }
+        } else {
+            indexFname = buildPath (binDistsPath, "Packages.gz");
+            if (!std.file.exists (indexFname))
+                indexFname = buildPath (binDistsPath, "Packages.xz");
+        }
+
+        if (indexFname.empty)
+            throw new Exception (format ("Couldn't download index file for %s/%s/%s", suite, section, arch));
+
         return indexFname;
     }
 

source/utils.d

@@ -19,6 +19,8 @@
 
 module ag.utils;
 
+import ag.logging;
+
 import std.stdio : writeln;
 import std.string;
 import std.ascii : letters, digits;
@@ -279,6 +281,38 @@ string getDataPath (string fname)
     return resPath;
 }
 
+@safe
+bool isRemote (const string uri)
+{
+    import std.regex;
+
+    auto uriregex = ctRegex!(`^(https?|ftps?)://`);
+
+    auto match = matchFirst(uri, uriregex);
+
+    return (!match.empty);
+}
+
+void downloadFile (const string url, const string dest)
+{
+    import std.file;
+    import std.path;
+    import std.net.curl;
+
+    if (dest.exists) {
+        logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
+        return;
+    }
+
+    mkdirRecurse (dest.dirName);
+
+    logDebug ("Downloading %s", url);
+    auto contents = get!(AutoProtocol, ubyte) (url);
+    logDebug ("Downloaded %s", url);
+
+    std.file.write (dest, contents);
+}
+
 unittest
 {
     writeln ("TEST: ", "GCID");
@@ -297,4 +331,7 @@ unittest
     assert (ImageSize (1024, 420).toInt () == 1024);
     assert (ImageSize (1024, 800) > ImageSize (64, 32));
     assert (ImageSize (48) < ImageSize (64));
+
+    assert (isRemoteUri ("http://test.com"));
+    assert (!isRemoteUri ("/srv/"));
 }