alpine: add capabilities to download packages via HTTP

src/asgen/backends/alpinelinux/apkindexutils.d

@@ -0,0 +1,150 @@
+module asgen.backends.alpinelinux.apkindexutils;
+
+import std.algorithm : remove;
+import std.algorithm.iteration : map;
+import std.algorithm.searching : canFind;
+import std.array : join, split;
+import std.file : exists;
+import std.format : format;
+import std.path : buildPath;
+import std.range : empty, InputRange, isForwardRange;
+import std.string : splitLines, startsWith, strip;
+import std.utf : validate;
+
+import std.stdio;
+
+import asgen.backends.alpinelinux.apkpkg;
+import asgen.downloader : Downloader, DownloadException;
+import asgen.logging : logDebug;
+import asgen.utils : isRemote;
+
+/**
+* Struct representing a block inside of an APKINDEX. Each block, seperated by
+* a newline, contains information about exactly one package.
+*/
+struct ApkIndexBlock {
+    string arch;
+    string maintainer;
+    string pkgname;
+    string pkgversion;
+    string pkgdesc;
+
+    @property string archiveName () {
+        return format ("%s-%s.apk", this.pkgname, this.pkgversion);
+    }
+}
+
+/**
+* Range for looping over the contents of an APKINDEX, block by block.
+*/
+struct ApkIndexBlockRange {
+    this (string contents)
+    {
+        this.lines = contents.splitLines;
+        this.getNextBlock();
+    }
+
+    @property ApkIndexBlock front () const {
+        return this.currentBlock;
+    }
+
+    @property bool empty () {
+        return this.m_empty;
+    }
+
+    void popFront ()
+    {
+        this.getNextBlock ();
+    }
+
+    @property ApkIndexBlockRange save () { return this; }
+
+private:
+    void getNextBlock () {
+        string[] completePair;
+        uint iterations = 0;
+
+        currentBlock = ApkIndexBlock();
+        foreach (currentLine; this.lines[this.lineDelta .. $]) {
+            iterations++;
+            if (currentLine == "") {
+                // next block for next package started
+                break;
+            } if (currentLine.canFind (":")) {
+                if (completePair.empty) {
+                    completePair = [currentLine];
+                    continue;
+                }
+
+                auto pair = completePair.join (" ").split (":");
+                this.setCurrentBlock (pair[0], pair[1]);
+                completePair = [currentLine];
+            } else {
+                completePair ~= currentLine.strip ();
+            }
+        }
+
+        this.lineDelta += iterations;
+        this.m_empty = this.lineDelta == this.lines.length;
+    }
+
+    void setCurrentBlock (string key, string value) {
+        switch (key) {
+            case "A":
+                this.currentBlock.arch = value;
+                break;
+            case "m":
+                this.currentBlock.maintainer = value;
+                break;
+            case "P":
+                this.currentBlock.pkgname = value;
+                break;
+            case "T":
+                this.currentBlock.pkgdesc = value;
+                break;
+            case "V":
+                this.currentBlock.pkgversion = value;
+                break;
+            default:
+                // We dont care about other keys
+                break;
+        }
+    }
+
+    string[] lines;
+    ApkIndexBlock currentBlock;
+    bool m_empty;
+    uint lineDelta;
+}
+
+static assert (isForwardRange!ApkIndexBlockRange);
+
+/**
+ * Download apkindex if required. Returns the path to the local copy of the APKINDEX.
+ */
+immutable (string) downloadIfNecessary (const string prefix,
+                                        const string destPrefix,
+                                        const string srcFileName,
+                                        const string destFileName)
+{
+    auto downloader = Downloader.get;
+
+    immutable filePath = buildPath (prefix, srcFileName);
+    immutable destFilePath = buildPath (destPrefix, destFileName);
+
+    if (filePath.isRemote) {
+        try {
+            downloader.downloadFile (filePath, destFilePath);
+
+            return destFilePath;
+        } catch (DownloadException e) {
+            logDebug ("Unable to download: %s", e.msg);
+        }
+    } else {
+        if (filePath.exists)
+            return filePath;
+    }
+
+    /* all extensions failed, so we failed */
+    throw new Exception ("Could not obtain any file matching %s".format (buildPath (prefix, srcFileName)));
+}

src/asgen/backends/alpinelinux/apkpkg.d

@@ -22,13 +22,15 @@
 
 module asgen.backends.alpinelinux.apkpkg;
 
-import std.stdio;
-import std.string;
 import std.array : empty;
+import std.format : format;
+import std.path : baseName, buildNormalizedPath, buildPath;
 
-import asgen.logging;
-import asgen.zarchive;
 import asgen.backends.interfaces;
+import asgen.config : Config;
+import asgen.downloader : Downloader;
+import asgen.utils : isRemote;
+import asgen.zarchive : ArchiveDecompressor;
 
 final class AlpinePackage : Package
 {
@@ -39,12 +41,24 @@ private:
     string pkgmaintainer;
     string[string] desc;
     string pkgFname;
+    string localPkgFName;
+    string tmpDir;
 
-    string[] contentsL;
+    string[] contentsL = null;
 
     ArchiveDecompressor archive;
 
 public:
+    this (string pkgname, string pkgver, string pkgarch)
+    {
+        this.pkgname = pkgname;
+        this.pkgver = pkgver;
+        this.pkgarch = pkgarch;
+
+        auto conf = Config.get ();
+        this.tmpDir = buildPath (conf.getTmpDir (), format ("%s-%s_%s", name, ver, arch));
+    }
+
     override @property string name () const
     {
         return this.pkgname;
@@ -85,9 +99,23 @@ public:
         this.pkgFname = fname;
     }
 
-    override @property string getFilename () const
+    override @property string getFilename ()
     {
-        return pkgFname;
+        if (!this.localPkgFName.empty)
+            return this.localPkgFName;
+
+        if (pkgFname.isRemote) {
+            synchronized (this) {
+                auto dl = Downloader.get;
+                immutable path = buildNormalizedPath (this.tmpDir, this.pkgFname.baseName);
+                dl.downloadFile (this.pkgFname, path);
+                this.localPkgFName = path;
+                return this.localPkgFName;
+            }
+        } else {
+            this.localPkgFName = pkgFname;
+            return this.localPkgFName;
+        }
     }
 
     override @property string maintainer () const
@@ -115,6 +143,13 @@ public:
 
     @property override string[] contents ()
     {
+        if (!this.contentsL.empty)
+            return this.contentsL;
+
+        ArchiveDecompressor ad;
+        ad.open (this.getFilename);
+        this.contentsL = ad.readContents ();
+
         return this.contentsL;
     }
 

src/asgen/backends/alpinelinux/apkpkgindex.d

@@ -19,36 +19,41 @@
 
 module asgen.backends.alpinelinux.apkpkgindex;
 
-import std.algorithm : canFind, filter, endsWith, remove;
-import std.array : appender, join, split;
+import std.array : appender;
 import std.conv : to;
 import std.exception : enforce;
-import std.file : dirEntries, exists, SpanMode;
+import std.file : exists;
 import std.format : format;
 import std.path : baseName, buildPath;
-import std.range : empty;
-import std.string : splitLines, startsWith, strip;
-import std.utf : UTFException, validate;
+import std.utf : validate;
 
-import asgen.logging;
-import asgen.zarchive;
-import asgen.utils : escapeXml;
+import asgen.config : Config;
+import asgen.logging : logError;
+import asgen.zarchive : ArchiveDecompressor;
+import asgen.utils : escapeXml, isRemote;
 import asgen.backends.interfaces;
 import asgen.backends.alpinelinux.apkpkg;
+import asgen.backends.alpinelinux.apkindexutils;
 
 final class AlpinePackageIndex : PackageIndex
 {
 
 private:
     string rootDir;
+    string tmpDir;
     Package[][string] pkgCache;
 
 public:
 
     this (string dir)
     {
-        enforce (exists (dir), format ("Directory '%s' does not exist.", dir));
+        if (!dir.isRemote)
+            enforce (exists (dir), format ("Directory '%s' does not exist.", dir));
+
         this.rootDir = dir;
+
+        auto conf = Config.get ();
+        tmpDir = buildPath (conf.getTmpDir, dir.baseName);
     }
 
     override void release ()
@@ -65,81 +70,30 @@ public:
         pkg.setDescription (desc, "C");
     }
 
-    private void setPkgValues (ref AlpinePackage pkg, string[] keyValueString)
-    {
-        immutable key = keyValueString[0].strip;
-        immutable value = keyValueString[1].strip;
-
-        switch (key) {
-        case "pkgname":
-            pkg.name = value;
-            break;
-        case "pkgver":
-            pkg.ver = value;
-            break;
-        case "arch":
-            pkg.arch = value;
-            break;
-        case "maintainer":
-            pkg.maintainer = value;
-            break;
-        case "pkgdesc":
-            setPkgDescription(pkg, value);
-            break;
-        default:
-            // We don't care about other entries
-            break;
-        }
-    }
-
     private Package[] loadPackages (string suite, string section, string arch)
     {
         auto apkRootPath = buildPath (rootDir, suite, section, arch);
-        ArchiveDecompressor ad;
+        auto indexFPath = downloadIfNecessary(apkRootPath, tmpDir, "APKINDEX.tar.gz", format("APKINDEX-%s-%s-%s.tar.gz", suite, section, arch));
         AlpinePackage[string] pkgsMap;
+        ArchiveDecompressor ad;
+        ad.open (indexFPath);
+        auto indexString = cast(string) ad.readData ("APKINDEX");
+        validate (indexString);
+        auto range = ApkIndexBlockRange (indexString);
 
-        foreach (packageArchivePath; dirEntries (apkRootPath, SpanMode.shallow).filter!(
-                f => f.name.endsWith (".apk"))) {
-            auto fileName = packageArchivePath.baseName ();
+        foreach (pkgInfo; range) {
+            auto fileName = pkgInfo.archiveName;
             AlpinePackage pkg;
             if (fileName in pkgsMap) {
                 pkg = pkgsMap[fileName];
             } else {
-                pkg = new AlpinePackage ();
+                pkg = new AlpinePackage (pkgInfo.pkgname, pkgInfo.pkgversion, pkgInfo.arch);
                 pkgsMap[fileName] = pkg;
             }
 
-            ad.open (packageArchivePath);
-            auto pkgInfoData = cast(string) ad.readData (".PKGINFO");
-
-            try {
-                validate (pkgInfoData);
-            } catch (UTFException e) {
-                logError ("PKGINFO file in archive %s contained invalid UTF-8, skipping!",
-                        packageArchivePath);
-                continue;
-            }
-
-            pkg.filename = packageArchivePath;
-            auto lines = pkgInfoData.splitLines ();
-            // If the current line doesn't contain a = it's meant to extend the previous line
-            string[] completePair;
-            foreach (currentLine; lines) {
-                if (currentLine.canFind ("=")) {
-                    if (completePair.empty) {
-                        completePair = [currentLine];
-                        continue;
-                    }
-
-                    this.setPkgValues (pkg, completePair.join (" ").split ("="));
-                    completePair = [currentLine];
-                } else if (!currentLine.startsWith ("#")) {
-                    completePair ~= currentLine.strip.split ("#")[0];
-                }
-            }
-            // We didn't process the last line yet
-            this.setPkgValues (pkg, completePair.join (" ").split ("="));
-            pkg.contents = ad.readContents ().remove!("a == \".PKGINFO\" || a.startsWith (\".SIGN\")");
+            pkg.filename = buildPath(rootDir, suite, section, arch, fileName);
+            pkg.maintainer = pkgInfo.maintainer;
+            setPkgDescription(pkg, pkgInfo.pkgdesc);
         }
 
         // perform a sanity check, so we will never emit invalid packages

src/asgen/meson.build

@@ -64,6 +64,7 @@ backend_sources = [
     'backends/dummy/pkgindex.d',
 
     'backends/alpinelinux/package.d',
+    'backends/alpinelinux/apkindexutils.d',
     'backends/alpinelinux/apkpkg.d',
     'backends/alpinelinux/apkpkgindex.d',