From 23284e62ba355b80f2f791112092de3f7578409d Mon Sep 17 00:00:00 2001
From: Walter Bright <walter@walterbright.com>
Date: Mon, 25 Mar 2024 23:33:47 -0700
Subject: [PATCH] Support library source file archives

---
 changelog/dmd.source-archive.dd |  97 +++++++++
 compiler/src/dmd/cli.d          |  12 ++
 compiler/src/dmd/common/file.d  | 117 +++++++++++
 compiler/src/dmd/file_manager.d | 345 ++++++++++++++++++++++++++++++--
 compiler/src/dmd/frontend.h     |  13 +-
 compiler/src/dmd/globals.d      |   3 +
 compiler/src/dmd/globals.h      |   4 +
 compiler/src/dmd/main.d         |   8 +
 compiler/src/dmd/mars.d         |  22 ++
 compiler/src/dmd/root/file.d    |   2 +-
 compiler/src/dmd/root/rmem.d    |   2 +-
 11 files changed, 603 insertions(+), 22 deletions(-)
 create mode 100644 changelog/dmd.source-archive.dd

diff --git a/changelog/dmd.source-archive.dd b/changelog/dmd.source-archive.dd
new file mode 100644
index 000000000000..4fdb61893ccc
--- /dev/null
+++ b/changelog/dmd.source-archive.dd
@@ -0,0 +1,97 @@
+# Support Source Archive Files
+
+This is a compiler feature, not a D language feature.
+
+Similar to how libraries of object files are made available to the linker,
+this adds source archive file support to the compiler. Any package (and all its
+sub-files) can become a source archive file. The source archive file is then
+supplied to the compiler rather than a directory with lots of files in it.
+
+This means, for example, that all of Phobos can be distributed as a single
+file, std.sar. (The .sar extension stands for "source archive".) If std.sar
+is in a path along the import path list supplied to the compiler, the
+compiler will prefer std.sar to looking through the std directory tree for the
+sub-modules. The std directory wouldn't even need to exist.
+
+The file format of the .sar file is very similar to that of object file libraries
+and various other schemes. It does not adhere to those other schemes due to their
+variances from platform to platform, all the code needed to support things that
+are unneeded for .sar files, and special consideration for D's needs. The format
+is meant to be friendly for memory-mapped file access, and does not have alignhment
+issues.
+
+A .sar file consists of the following sections:
+
+1. a header, to identify it as a .sar file with a magic number and a version
+
+2. a table of contents, one entry per source file. The entries consist of
+an offset/length to the filename string, and an offset/length to the file
+contents
+
+3. the filename strings, each string has a terminating 0
+
+4. the file contents, each file has four 0 bytes appended, as the lexer wants
+that as a sentinel
+
+5. the integers in the format are little-endian
+
+To create a .sar file, such as one for Phobos' std:
+
+dmd -sar=/home/duser/dmd/src/phobos/std
+
+and the file:
+
+/home/duser/dmd/src/phobos/std.sar
+
+will be created and filled with all sub-files with one of the extensions ".di", ".d",
+".c", or ".i". The filenames in the .sar file will include the path prefix
+specified in the -sar switch. This means the .sar file cannot be successfully moved
+to another location.
+
+For Phobos, std.sar is approximately 11 megabytes in size.
+
+To use the std.sar file, nothing needs to be changed in the user's build system.
+dmd will automatically prefer using any .sar file it finds. To disable using
+.sar files, which would be necessary when doing development of the source files,
+select one of the following:
+
+1. delete the .sar file
+
+2. use the -sar=off compiler switch. -sar=on turns it on, and is the default
+setting
+
+Trying out .sar with simple programs like hello world yield a negligible difference
+in compile speed. It's unlikely a larger program would show any particular trend
+in performance.
+
+A standalone archiver program can be easily created from the implementation in DMD.
+
+## Rationale
+
+1. All the source files in a project or library can be represented as a single file,
+making it easy to deal with.
+
+2. To compile all the source files at once with DMD, the command line can get
+extremely long, and certainly unwieldy. With .sar files, you may not even need
+a makefile or builder, just:
+
+dmd project.sar
+
+3. In Phobos (and most code), the tendency is to lump a lot of only marginally related
+functions into one file. This is likely because of the inconvenience of multiple files.
+In std.algorithm, for example, the individual algorithms could be placed into multiple
+files, since they don't refer to each other. This could also help the people who
+don't want the automatic "friend" status of declarations within a single module.
+.sar files can make much more granular modules more attractive.
+
+4. A directory in a project can be anything. But multiple .sar files in a project means
+that's where the code is. Multiple versions of the project can exist in the same directory
+by using .sar files with different names.
+
+5. Experiments with small programs (hello world) show a negligible change in compile
+speed. Much larger programs may show significant compile speed increases with .sar
+files, simply because there are fewer file operations. For slow file systems, such as
+SD cards, or network file systems, the speedup could be substantial.
+
+None of these make it a slam-dunk, after all, no other compiler does this that I'm
+aware of. Even so, some surprising uses can be expected of .sar files.
diff --git a/compiler/src/dmd/cli.d b/compiler/src/dmd/cli.d
index 24cf50729b23..687679a3eea3 100644
--- a/compiler/src/dmd/cli.d
+++ b/compiler/src/dmd/cli.d
@@ -766,6 +766,18 @@ dmd -cov -unittest myprog.d
             `$(UNIX Generate shared library)
              $(WINDOWS Generate DLL library)`,
         ),
+        Option("sar=[on|off|<path/package>]",
+            "turn reading source archive files on or off, or create source archive at <path/package.sar>",
+            `Controls source archive files and usage.
+                $(UL
+                    $(LI $(I on): use source archive files (default))
+                    $(LI $(I off): ignore source archive files)
+                    $(LI $(I path/package): create source archive file.
+                    <path> is where the root package of the files to be archived are.
+                    All the modules in <package> are written to the source archive file <path/package.sar>.
+                    Do not use in combination with compiling, as that will be very slow.)
+                )`
+        ),
         Option("target=<triple>",
                "use <triple> as <arch>-[<vendor>-]<os>[-<cenv>[-<cppenv]]",
                "$(I arch) is the architecture: either `x86`, `x64`, `x86_64` or `x32`,
diff --git a/compiler/src/dmd/common/file.d b/compiler/src/dmd/common/file.d
index 80677f66ff89..1dd94f5e89bf 100644
--- a/compiler/src/dmd/common/file.d
+++ b/compiler/src/dmd/common/file.d
@@ -14,12 +14,18 @@
 
 module dmd.common.file;
 
+import core.stdc.stdio;
+import core.stdc.stdlib;
+import core.stdc.string;
+import core.stdc.limits;
+
 import core.stdc.errno : errno;
 import core.stdc.stdio : fprintf, remove, rename, stderr;
 import core.stdc.stdlib;
 import core.stdc.string : strerror, strlen, memcpy;
 
 import dmd.common.smallbuffer;
+import dmd.root.filename;
 
 version (Windows)
 {
@@ -32,6 +38,7 @@ version (Windows)
 }
 else version (Posix)
 {
+    import core.sys.posix.dirent;
     import core.sys.posix.fcntl;
     import core.sys.posix.sys.mman;
     import core.sys.posix.sys.stat;
@@ -587,3 +594,113 @@ private auto ref fakePure(F)(scope F fun) pure
     mixin("alias PureFun = " ~ F.stringof ~ " pure;");
     return (cast(PureFun) fun)();
 }
+
+/***********************************
+ * Recursively search all the directories and files under dir_path
+ * for files that match one of the extensions in exts[].
+ * Pass the matches to sink.
+ * Params:
+ *      dir_path = root of directories to search
+ *      exts = array of filename extensions to match
+ *      filenameSink = accepts the resulting matches
+ * Returns:
+ *      true for failed to open the directory
+ */
+bool find_files(const char* dir_path, const char[][] exts, void delegate(const(char)[]) nothrow filenameSink)
+{
+    //printf("find_files() dir_path: %s\n", dir_path);
+    version (Windows)
+    {
+        char[MAX_PATH + 1] full_path = void;
+        snprintf(full_path.ptr, full_path.length, "%s\\*", dir_path);
+
+        WIN32_FIND_DATAA ffd;
+        HANDLE hFind = FindFirstFileA(full_path.ptr, &ffd);
+
+        if (hFind == INVALID_HANDLE_VALUE)
+            return true;
+
+        do
+        {
+            if (!(ffd.cFileName == "." || ffd.cFileName == ".."))
+                continue;
+
+            snprintf(full_path.ptr, full_path.length, "%s\\%s", dir_path, ffd.cFileName.ptr);
+
+            const(char)[] name = ffd.cFileName.ptr[0 .. strlen(ffd.cFileName.ptr)]; // convert to D string
+            if (!name.length)
+                continue; // ignore
+
+            if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+            {
+                if (!(name == "." || name == ".."))
+                    find_files(full_path.ptr, exts, filenameSink);
+            }
+            else
+            {
+                if (name[0] == '.')
+                    continue;           // ignore files that start with a .
+                foreach (ext; exts[])
+                {
+                    if (FileName.ext(name) == ext)
+                    {
+                        //printf("%s\n", full_path.ptr);
+                        filenameSink(full_path[0 .. strlen(full_path.ptr)]);
+                    }
+                }
+            }
+        } while (FindNextFileA(hFind, &ffd) != 0);
+
+        FindClose(hFind);
+        return false;
+    }
+    else version (Posix)
+    {
+        DIR* dir = opendir(dir_path);
+        if (!dir)
+            return true;
+
+        dirent* entry;
+        while ((entry = readdir(dir)) != null)
+        {
+            // PATH_MAX apparently is not in core.std.limits in some bootstrap compilers
+            // and dstyle claims it "is never used"
+            //enum PATH_MAX = 1024;
+
+            char[1024/*PATH_MAX*/ + 1] full_path = void;
+            snprintf(full_path.ptr, full_path.length, "%s/%s", dir_path, entry.d_name.ptr);
+
+            stat_t statbuf;
+            if (lstat(full_path.ptr, &statbuf) == -1)
+                continue;
+
+            const(char)[] name = entry.d_name.ptr[0 .. strlen(entry.d_name.ptr)]; // convert to D string
+            if (!name.length)
+                continue; // ignore
+
+            if (S_ISDIR(statbuf.st_mode))
+            {
+                if (!(name == "." || name == ".."))
+                    find_files(full_path.ptr, exts, filenameSink);
+            }
+            else if (S_ISREG(statbuf.st_mode))
+            {
+                if (name[0] == '.')
+                    continue;           // ignore files that start with a .
+                foreach (ext; exts)
+                {
+                    if (FileName.ext(name) == ext)
+                    {
+                        //printf("%s\n", full_path.ptr);
+                        filenameSink(full_path[0 .. strlen(full_path.ptr)]);
+                    }
+                }
+            }
+        }
+
+        closedir(dir);
+        return false;
+    }
+    else
+        static assert(0);
+}
diff --git a/compiler/src/dmd/file_manager.d b/compiler/src/dmd/file_manager.d
index 6ad3d621708d..864834930321 100644
--- a/compiler/src/dmd/file_manager.d
+++ b/compiler/src/dmd/file_manager.d
@@ -11,10 +11,15 @@
 module dmd.file_manager;
 
 import core.stdc.stdio;
+import core.stdc.string;
+
+import dmd.common.file;
 import dmd.common.outbuffer;
+import dmd.root.array;
 import dmd.root.stringtable : StringTable;
 import dmd.root.file : File, Buffer;
 import dmd.root.filename : FileName, isDirSeparator;
+import dmd.root.rmem;
 import dmd.root.string : toDString;
 import dmd.errors;
 import dmd.globals;
@@ -23,6 +28,7 @@ import dmd.location;
 
 enum package_d  = "package." ~ mars_ext;
 enum package_di = "package." ~ hdr_ext;
+enum ext_sar    = "sar";        // file extension for source archive
 
 /// Returns: whether a file with `name` is a special "package.d" module
 bool isPackageFileName(scope FileName fileName) nothrow
@@ -91,10 +97,11 @@ private struct PathCache
      * Cache the results for the path and each left-justified subpath of the path.
      * Params:
      *  filespec = path/filename
+     *  fileManager = where the caches are
      * Returns:
      *  true if path exists, false if it does not
      */
-    bool pathExists(const(char)[] filespec) nothrow
+    bool pathExists(const(char)[] filespec, FileManager fileManager) nothrow
     {
         /* look for the longest leftmost parent path that is cached
          * by starting at the right and working to the left
@@ -116,12 +123,57 @@ private struct PathCache
             if (!exists)
                 pathStatus.insert(st.cur, false);
             else
-                exists = pathStatus.insert(st.cur, FileName.exists(st.cur) == 2).value;
+            {
+                if (fileManager.useSourceArchive)
+                {
+                    /* See if .sar file for this directory exists, if so read it
+                     */
+                    //printf("st.cur: %.*s\n", cast(int)st.cur.length, st.cur.ptr);
+                    const(char)[] name = FileName.name(st.cur);
+                    const(char)[] path = st.cur.length == name.length ? null : st.cur[0 .. $ - (name.length + 1)];
+                    if (readSourceArchive(fileManager, path, name))
+                    {
+                        // pathStatus is updated, too, along with everything to the right
+                        exists = true;
+                        break;
+                    }
+                }
+                //printf("pathStatus.insert %.*s\n", cast(int)st.cur.length, st.cur.ptr);
+                //printf("pathStatus.insert %s\n", st.cur.ptr);
+                //printf("FileName.exists() %d\n", FileName.exists(st.cur.ptr));
+                auto cached = pathStatus.insert(st.cur, FileName.exists(st.cur) == 2);
+                if (cached)
+                    exists = cached.value;
+                else
+                    exists = true;
+            }
         }
 
         return exists;
     }
 
+    /**
+     * Mark path and each left-justified subpath as existing in the cache.
+     * Params:
+     *  filespec = path/filename
+     */
+    void addPath(const(char)[] filespec) nothrow
+    {
+        /* look for the longest leftmost parent path
+         * by starting at the right and working to the left
+         */
+        auto st = PathStack(filespec);
+        while (st.up)
+        {
+            if (auto cached = pathStatus.lookup(st.cur))
+            {
+                if (cached.value)   // subpath exists
+                    break;          // assume any more to the left also exist
+            }
+            pathStatus.insert(st.cur, true);
+        }
+    }
+
     /**
      * Ask if path ends in a directory.
      * Cache result for speed.
@@ -139,20 +191,37 @@ private struct PathCache
     }
 }
 
+/**************************************************************
+ */
+
 final class FileManager
 {
     private StringTable!(const(ubyte)[]) files;  // contents of files indexed by file name
 
     private PathCache pathCache;
 
+    private bool useSourceArchive;         // use .sar files if they exist
+
     ///
-    public this () nothrow
+    public this() nothrow
     {
         this.files._init();
         this.pathCache.pathStatus._init();
     }
 
 nothrow:
+
+    /**********************************
+     * Set useSourceArchive flag
+     * Params:
+     *  useSourceArchive = value to set it to
+     */
+    public void setUseSourceArchive(bool useSourceArchive) nothrow
+    {
+        //printf("setUseSourceArchive: %d\n", useSourceArchive);
+        this.useSourceArchive = useSourceArchive;
+    }
+
     /********************************************
     * Look for the source file if it's different from filename.
     * Look for .di, .d, directory, and along global.path.
@@ -169,10 +238,25 @@ nothrow:
         //printf("lookForSourceFile(`%.*s`)\n", cast(int)filename.length, filename.ptr);
         /* Search along paths[] for .di file, then .d file.
         */
+
+        /******************
+         * Check if file exists in the source archive or on disk.
+         * Params:
+         *      filename = fqn of file
+         * Returns:
+         *      true if it exists
+         */
+        bool isFileExisting(const char[] filename)
+        {
+            if (useSourceArchive && files.lookup(filename))
+                return true;
+            return FileName.exists(filename) == 1;
+        }
+
         // see if we should check for the module locally.
-        bool checkLocal = pathCache.pathExists(filename);
+        bool checkLocal = pathCache.pathExists(filename, this);
         const sdi = FileName.forceExt(filename, hdr_ext);
-        if (checkLocal && FileName.exists(sdi) == 1)
+        if (checkLocal && isFileExisting(sdi))
             return sdi;
         scope(exit) FileName.free(sdi.ptr);
 
@@ -180,7 +264,7 @@ nothrow:
         // Special file name representing `stdin`, always assume its presence
         if (sd == "__stdin.d")
             return sd;
-        if (checkLocal && FileName.exists(sd) == 1)
+        if (checkLocal && isFileExisting(sd))
             return sd;
         scope(exit) FileName.free(sd.ptr);
 
@@ -193,12 +277,12 @@ nothrow:
                  * iff filename/package.d is a file
                  */
                 const ni = FileName.combine(filename, package_di);
-                if (FileName.exists(ni) == 1)
+                if (isFileExisting(ni))
                     return ni;
                 FileName.free(ni.ptr);
 
                 const n = FileName.combine(filename, package_d);
-                if (FileName.exists(n) == 1)
+                if (isFileExisting(n))
                     return n;
                 FileName.free(n.ptr);
             }
@@ -211,20 +295,19 @@ nothrow:
         foreach (entry; paths)
         {
             const p = entry.toDString();
-
             const(char)[] n = FileName.combine(p, sdi);
 
-            if (!pathCache.pathExists(n)) {
+            if (!pathCache.pathExists(n, this)) {
                 FileName.free(n.ptr);
                 continue; // no need to check for anything else.
             }
-            if (FileName.exists(n) == 1) {
+            if (isFileExisting(n)) {
                 return n;
             }
             FileName.free(n.ptr);
 
             n = FileName.combine(p, sd);
-            if (FileName.exists(n) == 1) {
+            if (isFileExisting(n)) {
                 return n;
             }
             FileName.free(n.ptr);
@@ -236,11 +319,11 @@ nothrow:
             if (pathCache.isExistingPath(n))
             {
                 const n2i = FileName.combine(n, package_di);
-                if (FileName.exists(n2i) == 1)
+                if (isFileExisting(n2i))
                     return n2i;
                 FileName.free(n2i.ptr);
                 const n2 = FileName.combine(n, package_d);
-                if (FileName.exists(n2) == 1) {
+                if (isFileExisting(n2)) {
                     return n2;
                 }
                 FileName.free(n2.ptr);
@@ -250,12 +333,12 @@ nothrow:
         /* ImportC: No D modules found, now search along paths[] for .i file, then .c file.
          */
         const si = FileName.forceExt(filename, i_ext);
-        if (FileName.exists(si) == 1)
+        if (isFileExisting(si))
             return si;
         scope(exit) FileName.free(si.ptr);
 
         const sc = FileName.forceExt(filename, c_ext);
-        if (FileName.exists(sc) == 1)
+        if (isFileExisting(sc))
             return sc;
         scope(exit) FileName.free(sc.ptr);
         foreach (entry; paths)
@@ -263,13 +346,13 @@ nothrow:
             const p = entry.toDString();
 
             const(char)[] n = FileName.combine(p, si);
-            if (FileName.exists(n) == 1) {
+            if (isFileExisting(n)) {
                 return n;
             }
             FileName.free(n.ptr);
 
             n = FileName.combine(p, sc);
-            if (FileName.exists(n) == 1) {
+            if (isFileExisting(n)) {
                 return n;
             }
             FileName.free(n.ptr);
@@ -290,7 +373,10 @@ nothrow:
     {
         const name = filename.toString;
         if (auto val = files.lookup(name))      // if `name` is cached
+        {
+            //printf("File.read() cached %.*s, %p[%d]\n", cast(int)name.length, name.ptr, val.value.ptr, cast(int)val.value.length);
             return val.value;                   // return its contents
+        }
 
         OutBuffer buf;
         if (name == "__stdin.d")                // special name for reading from stdin
@@ -303,6 +389,7 @@ nothrow:
             if (FileName.exists(name) != 1) // if not an ordinary file
                 return null;
 
+            //printf("File.read() %.*s\n", cast(int)name.length, name.ptr);
             if (File.read(name, buf))
                 return null;        // failed
         }
@@ -330,6 +417,19 @@ nothrow:
         auto val = files.insert(filename.toString, buffer);
         return val == null ? null : val.value;
     }
+
+    /**
+     * Add the file's name and contents to the cache.
+     * Params:
+     *  fileName = fqn of the file
+     *  contents = the file contents
+     */
+    void addFileNameAndContents(const(char)[] fileName, const(ubyte)[] contents)
+    {
+        //printf("addFileNameAndContents() %.*s\n", cast(int)fileName.length, fileName.ptr);
+        pathCache.addPath(fileName);
+        files.insert(fileName, contents);
+    }
 }
 
 private bool readFromStdin(ref OutBuffer sink) nothrow
@@ -364,3 +464,212 @@ private bool readFromStdin(ref OutBuffer sink) nothrow
 
     assert(0);
 }
+
+/********************************************
+ * Read all the modules, and build one giant cache file out of them.
+ * Params:
+ *      pathPackage = the path/package to build a cache file for
+ * Returns:
+ *      true = failed
+ *      false = success
+ */
+bool writeSourceArchive(const(char)[] pathPackage)
+{
+    enum log = false;
+    if (log) printf("writeSourceArchive() %.*s\n", cast(int)pathPackage.length, pathPackage.ptr);
+
+    const(char)[] name = FileName.name(pathPackage);
+    const size_t nameStart = pathPackage.length - name.length;
+
+    Array!(const(char)*) fileNames;
+
+    void accumulate(scope const(char)[] filename) nothrow
+    {
+        if (log) printf("%.*s\n", cast(int)filename.length, filename.ptr);
+        fileNames.push(xarraydup(filename).ptr);
+    }
+
+    immutable string[4] exts = ["d", "di", "c", "i"];   // archive files with these extensions
+
+    const(char)[] dir_path = xarraydup(pathPackage);
+
+    if (find_files(dir_path.ptr, exts, &accumulate))
+        return true;
+
+    OutBuffer ar;  // ar will hold the contents of the archive file
+
+    SrcArchiveHeader srcArchiveHeader;
+    srcArchiveHeader.contentsOffset = cast(uint)SrcArchiveHeader.sizeof;
+    srcArchiveHeader.contentsLength = cast(uint)fileNames.length;
+
+    ar.write(&srcArchiveHeader, srcArchiveHeader.sizeof);
+
+    /* Allocate in ar the array of Content
+     */
+    size_t length = fileNames.length;
+    ar.allocate(length * Content.sizeof);
+
+    /* Temporary array of Content
+     */
+    Content[] contents = (cast(Content*)mem.xmalloc(length * Content.sizeof))[0 .. length];
+
+    /* Write the name strings to ar
+     */
+    foreach (i, ref content; contents[])
+    {
+        const(char)* fnp = fileNames[i];
+        const(char)[] fn = fnp[nameStart .. strlen(fnp)];       // slice off path/ prefix
+        if (log) printf("fn: %.*s\n", cast(int)fn.length, fn.ptr);
+
+        content.nameOffset = cast(uint)ar.length;
+        content.nameLength = cast(uint)fn.length;
+        //printf("1content.nameOffset[%lld]: %d nameLength: %d\n", i, content.nameOffset, content.nameLength);
+
+        ar.writeStringz(fn);
+    }
+
+    /* Read the files, and write their contents to ar, along with
+     * a terminating 4 bytes of 0 (to accommodate dchar source files)
+     */
+    OutBuffer fb;
+    foreach (i, ref content; contents[])
+    {
+        fb.reset();                          // recycle read buffer
+        File.read(fileNames[i][0 .. strlen(fileNames[i])], fb); // read file into fb
+        content.importOffset = cast(uint)ar.length;
+        content.importLength = cast(uint)fb.length; // don't include terminating 0
+        ar.write(fb.peekSlice());                   // append file contents to ar
+        ar.write32(0);                              // append 4 bytes of 0
+    }
+
+    /* Copy contents[] into ar
+     */
+    Content[] cfcontents = (cast(Content*)(ar.peekSlice().ptr + srcArchiveHeader.contentsOffset))[0 .. length];
+    cfcontents[] = contents[];
+    mem.xfree(contents.ptr);
+
+    /* Create source archive file name by appending ".sar" to pathPackage
+     */
+    auto sourceArchiveFileName = FileName.addExt(pathPackage, ext_sar);
+
+    /* write the source archive file
+     */
+    if (log) printf("writing sourceArchive file %.*s\n", cast(int)sourceArchiveFileName.length, sourceArchiveFileName.ptr);
+    if (!writeFile(sourceArchiveFileName.ptr, ar.peekSlice()))
+        return true;    // failure
+
+    return false; // success
+}
+
+/**********************************************
+ * Read the source archive file, making it a memory mapped file.
+ * Fill the FileManager.pathCache.pathStatus for each of the paths in the source archive,
+ * and the contents of the files in the archive go in FileManager.files.
+ * Since it is a memory mapped file, the contents are only read when the module is
+ * actually imported.
+ * Params:
+ *      fileManager = manages files
+ *      path = an import path
+ *      pkg = a package name the path leads to
+ * Returns:
+ *      true = success
+ *      false = failure
+ */
+nothrow
+bool readSourceArchive(FileManager fileManager, const(char)[] path, const(char)[] pkg)
+{
+    enum log = false;
+    if (log) printf("readSourceArchive(path: %.*s, pkg: %.*s)\n", cast(int)path.length, path.ptr, cast(int)pkg.length, pkg.ptr);
+
+    /* Combine path, pkg, and .sar
+     */
+    const(char)[] pathPackage = FileName.combine(path, pkg);
+    const(char)[] sourceArchiveFileName = FileName.addExt(pathPackage, ext_sar);
+
+    /* Open memory mapped file on sourceArchiveFileName
+     */
+    static if (1) // memory mapped file
+    {
+        auto mmFile = new FileMapping!(const char)(sourceArchiveFileName.ptr);
+        auto data = (*mmFile)[];    // all the data in the file
+    }
+    else // regular file read
+    {
+        OutBuffer buf;
+        if (!File.read(sourceArchiveFileName, buf))
+        {
+            if (log) printf("no .sar file\n");
+            return false;           // empty files are ok
+        }
+        auto data = buf.extractSlice(true);    // all the data in the file
+    }
+    if (data.length == 0)
+    {
+        if (log) printf(" no .sar file\n");
+        return false;           // empty files are ok
+    }
+
+    SrcArchiveHeader* srcArchiveHeader = cast(SrcArchiveHeader*)data.ptr;
+    if (data.length < SrcArchiveHeader.sizeof ||
+        srcArchiveHeader.magicNumber != SrcArchiveHeader.MagicNumber ||
+        srcArchiveHeader.versionNumber != 1 ||
+        data.length < srcArchiveHeader.contentsOffset ||
+        srcArchiveHeader.contentsLength >= uint.max / Content.sizeof - srcArchiveHeader.contentsOffset || // overflow check
+        data.length < srcArchiveHeader.contentsOffset + srcArchiveHeader.contentsLength * Content.sizeof)
+    {
+        printf("corrupt .sar file %d\n", __LINE__);
+        return false;    // corrupt file
+    }
+
+    foreach (i; 0 .. srcArchiveHeader.contentsLength)
+    {
+        const Content* cp = cast(const(Content)*)(data.ptr + srcArchiveHeader.contentsOffset + i * Content.sizeof);
+
+        if (data.length <= cp.nameOffset ||
+            cp.nameOffset >= cp.nameOffset + cp.nameLength + 1 ||
+            data.length <= cp.nameOffset + cp.nameLength + 1 ||
+            data.length <= cp.importOffset ||
+            cp.importOffset >= cp.importOffset + cp.importLength + 1 ||
+            data.length <= cp.importOffset + cp.importLength + 1)
+        {
+            printf("corrupt .sar file %d\n", __LINE__);
+            return false;    // corrupt file
+        }
+
+        const(char)[] fileName = (data.ptr + cp.nameOffset)[0 .. cp.nameLength];
+        const(ubyte)[] fileContents = (cast(const(ubyte)*)(data.ptr + cp.importOffset))[0 .. cp.importLength];
+
+        //printf("fileContents: %s %p[%lld]\n", fileName.ptr, fileContents.ptr, fileContents.length);
+        //printf("fileContents: %s\n", fileContents.ptr);
+
+        // Cache file name and file contents (but don't read the file contents!)
+        const(char)[] fqn = FileName.combine(path, fileName);
+        fileManager.addFileNameAndContents(fqn, fileContents);
+        if (log) printf("fqn: %.*s\n", cast(int)fqn.length, fqn.ptr);
+    }
+    if (log) printf("done reading .sar file\n");
+
+    return true;
+}
+
+/***************************
+ * The source archive file starts with this header.
+ */
+struct SrcArchiveHeader
+{
+    enum MagicNumber = 0x64FE_ED63;
+    uint magicNumber = MagicNumber;     // don't collide with other file types
+    uint versionNumber = 1;             // so we can change the format
+    uint contentsOffset;                // file offset to start of contents section
+    uint contentsLength;                // the number of contents in the contents section
+}
+
+/* The contents section is an array of Contents
+ */
+struct Content
+{
+    uint nameOffset;    // file offset to name of file
+    uint nameLength;    // number of characters in name, excluding terminating 0
+    uint importOffset;  // file offset to start of imported file contents
+    uint importLength;  // number of characters in the imported file contents, excluding terminating 0
+}
diff --git a/compiler/src/dmd/frontend.h b/compiler/src/dmd/frontend.h
index 22e02c382f29..7fc91e4dd748 100644
--- a/compiler/src/dmd/frontend.h
+++ b/compiler/src/dmd/frontend.h
@@ -7938,6 +7938,7 @@ struct Help final
     bool checkAction;
     bool revert;
     bool preview;
+    bool sar;
     bool externStd;
     bool hc;
     Help() :
@@ -7949,11 +7950,12 @@ struct Help final
         checkAction(),
         revert(),
         preview(),
+        sar(),
         externStd(),
         hc()
     {
     }
-    Help(bool manual, bool usage = false, bool mcpu = false, bool transition = false, bool check = false, bool checkAction = false, bool revert = false, bool preview = false, bool externStd = false, bool hc = false) :
+    Help(bool manual, bool usage = false, bool mcpu = false, bool transition = false, bool check = false, bool checkAction = false, bool revert = false, bool preview = false, bool sar = false, bool externStd = false, bool hc = false) :
         manual(manual),
         usage(usage),
         mcpu(mcpu),
@@ -7962,6 +7964,7 @@ struct Help final
         checkAction(checkAction),
         revert(revert),
         preview(preview),
+        sar(sar),
         externStd(externStd),
         hc(hc)
         {}
@@ -8053,6 +8056,7 @@ struct Param final
     bool addMain;
     bool allInst;
     bool bitfields;
+    bool useSourceArchive;
     CppStdRevision cplusplus;
     Help help;
     Verbose v;
@@ -8088,6 +8092,7 @@ struct Param final
     _d_dynamicArray< const char > objdir;
     _d_dynamicArray< const char > objname;
     _d_dynamicArray< const char > libname;
+    _d_dynamicArray< const char > pathPackage;
     Output ddoc;
     Output dihdr;
     Output cxxhdr;
@@ -8136,6 +8141,7 @@ struct Param final
         addMain(),
         allInst(),
         bitfields(),
+        useSourceArchive(true),
         cplusplus((CppStdRevision)201103u),
         help(),
         v(),
@@ -8165,6 +8171,7 @@ struct Param final
         objdir(),
         objname(),
         libname(),
+        pathPackage(),
         ddoc(),
         dihdr(),
         cxxhdr(),
@@ -8189,7 +8196,7 @@ struct Param final
         mapfile()
     {
     }
-    Param(bool obj, bool multiobj = false, bool trace = false, bool tracegc = false, bool vcg_ast = false, DiagnosticReporting useDeprecated = (DiagnosticReporting)1u, bool useUnitTests = false, bool useInline = false, bool release = false, bool preservePaths = false, DiagnosticReporting warnings = (DiagnosticReporting)2u, bool cov = false, uint8_t covPercent = 0u, bool ctfe_cov = false, bool ignoreUnsupportedPragmas = true, bool useModuleInfo = true, bool useTypeInfo = true, bool useExceptions = true, bool useGC = true, bool betterC = false, bool addMain = false, bool allInst = false, bool bitfields = false, CppStdRevision cplusplus = (CppStdRevision)201103u, Help help = Help(), Verbose v = Verbose(), FeatureState useDIP25 = (FeatureState)2u, FeatureState useDIP1000 = (FeatureState)0u, bool ehnogc = false, bool useDIP1021 = false, FeatureState fieldwise = (FeatureState)0u, bool fixAliasThis = false, FeatureState rvalueRefParam = (FeatureState)0u, FeatureState noSharedAccess = (FeatureState)0u, bool previewIn = false, bool inclusiveInContracts = false, bool shortenedMethods = true, bool fixImmutableConv = false, bool fix16997 = true, FeatureState dtorFields = (FeatureState)0u, FeatureState systemVariables = (FeatureState)0u, CHECKENABLE useInvariants = (CHECKENABLE)0u, CHECKENABLE useIn = (CHECKENABLE)0u, CHECKENABLE useOut = (CHECKENABLE)0u, CHECKENABLE useArrayBounds = (CHECKENABLE)0u, CHECKENABLE useAssert = (CHECKENABLE)0u, CHECKENABLE useSwitchError = (CHECKENABLE)0u, CHECKENABLE boundscheck = (CHECKENABLE)0u, CHECKACTION checkAction = (CHECKACTION)0u, CLIIdentifierTable dIdentifierTable = (CLIIdentifierTable)0u, CLIIdentifierTable cIdentifierTable = (CLIIdentifierTable)0u, _d_dynamicArray< const char > argv0 = {}, Array<const char* > modFileAliasStrings = Array<const char* >(), Array<const char* > imppath = Array<const char* >(), Array<const char* > fileImppath = Array<const char* >(), _d_dynamicArray< const char > objdir = {}, _d_dynamicArray< const char > objname = {}, _d_dynamicArray< const char > libname = {}, Output ddoc = Output(), Output dihdr = Output(), Output cxxhdr = Output(), Output json = Output(), JsonFieldFlags jsonFieldFlags = (JsonFieldFlags)0u, Output makeDeps = Output(), Output mixinOut = Output(), Output moduleDeps = Output(), uint32_t debuglevel = 0u, uint32_t versionlevel = 0u, bool run = false, Array<const char* > runargs = Array<const char* >(), Array<const char* > cppswitches = Array<const char* >(), const char* cpp = nullptr, Array<const char* > objfiles = Array<const char* >(), Array<const char* > linkswitches = Array<const char* >(), Array<bool > linkswitchIsForCC = Array<bool >(), Array<const char* > libfiles = Array<const char* >(), Array<const char* > dllfiles = Array<const char* >(), _d_dynamicArray< const char > deffile = {}, _d_dynamicArray< const char > resfile = {}, _d_dynamicArray< const char > exefile = {}, _d_dynamicArray< const char > mapfile = {}) :
+    Param(bool obj, bool multiobj = false, bool trace = false, bool tracegc = false, bool vcg_ast = false, DiagnosticReporting useDeprecated = (DiagnosticReporting)1u, bool useUnitTests = false, bool useInline = false, bool release = false, bool preservePaths = false, DiagnosticReporting warnings = (DiagnosticReporting)2u, bool cov = false, uint8_t covPercent = 0u, bool ctfe_cov = false, bool ignoreUnsupportedPragmas = true, bool useModuleInfo = true, bool useTypeInfo = true, bool useExceptions = true, bool useGC = true, bool betterC = false, bool addMain = false, bool allInst = false, bool bitfields = false, bool useSourceArchive = true, CppStdRevision cplusplus = (CppStdRevision)201103u, Help help = Help(), Verbose v = Verbose(), FeatureState useDIP25 = (FeatureState)2u, FeatureState useDIP1000 = (FeatureState)0u, bool ehnogc = false, bool useDIP1021 = false, FeatureState fieldwise = (FeatureState)0u, bool fixAliasThis = false, FeatureState rvalueRefParam = (FeatureState)0u, FeatureState noSharedAccess = (FeatureState)0u, bool previewIn = false, bool inclusiveInContracts = false, bool shortenedMethods = true, bool fixImmutableConv = false, bool fix16997 = true, FeatureState dtorFields = (FeatureState)0u, FeatureState systemVariables = (FeatureState)0u, CHECKENABLE useInvariants = (CHECKENABLE)0u, CHECKENABLE useIn = (CHECKENABLE)0u, CHECKENABLE useOut = (CHECKENABLE)0u, CHECKENABLE useArrayBounds = (CHECKENABLE)0u, CHECKENABLE useAssert = (CHECKENABLE)0u, CHECKENABLE useSwitchError = (CHECKENABLE)0u, CHECKENABLE boundscheck = (CHECKENABLE)0u, CHECKACTION checkAction = (CHECKACTION)0u, CLIIdentifierTable dIdentifierTable = (CLIIdentifierTable)0u, CLIIdentifierTable cIdentifierTable = (CLIIdentifierTable)0u, _d_dynamicArray< const char > argv0 = {}, Array<const char* > modFileAliasStrings = Array<const char* >(), Array<const char* > imppath = Array<const char* >(), Array<const char* > fileImppath = Array<const char* >(), _d_dynamicArray< const char > objdir = {}, _d_dynamicArray< const char > objname = {}, _d_dynamicArray< const char > libname = {}, _d_dynamicArray< const char > pathPackage = {}, Output ddoc = Output(), Output dihdr = Output(), Output cxxhdr = Output(), Output json = Output(), JsonFieldFlags jsonFieldFlags = (JsonFieldFlags)0u, Output makeDeps = Output(), Output mixinOut = Output(), Output moduleDeps = Output(), uint32_t debuglevel = 0u, uint32_t versionlevel = 0u, bool run = false, Array<const char* > runargs = Array<const char* >(), Array<const char* > cppswitches = Array<const char* >(), const char* cpp = nullptr, Array<const char* > objfiles = Array<const char* >(), Array<const char* > linkswitches = Array<const char* >(), Array<bool > linkswitchIsForCC = Array<bool >(), Array<const char* > libfiles = Array<const char* >(), Array<const char* > dllfiles = Array<const char* >(), _d_dynamicArray< const char > deffile = {}, _d_dynamicArray< const char > resfile = {}, _d_dynamicArray< const char > exefile = {}, _d_dynamicArray< const char > mapfile = {}) :
         obj(obj),
         multiobj(multiobj),
         trace(trace),
@@ -8213,6 +8220,7 @@ struct Param final
         addMain(addMain),
         allInst(allInst),
         bitfields(bitfields),
+        useSourceArchive(useSourceArchive),
         cplusplus(cplusplus),
         help(help),
         v(v),
@@ -8248,6 +8256,7 @@ struct Param final
         objdir(objdir),
         objname(objname),
         libname(libname),
+        pathPackage(pathPackage),
         ddoc(ddoc),
         dihdr(dihdr),
         cxxhdr(cxxhdr),
diff --git a/compiler/src/dmd/globals.d b/compiler/src/dmd/globals.d
index f8291fa5ce7e..b1af4d220342 100644
--- a/compiler/src/dmd/globals.d
+++ b/compiler/src/dmd/globals.d
@@ -108,6 +108,7 @@ extern(C++) struct Help
     bool checkAction;  // -checkaction
     bool revert;       // -revert
     bool preview;      // -preview
+    bool sar;          // -sar
     bool externStd;    // -extern-std
     bool hc;           // -HC
 }
@@ -170,6 +171,7 @@ extern (C++) struct Param
     bool addMain;           // add a default main() function
     bool allInst;           // generate code for all template instantiations
     bool bitfields;         // support C style bit fields
+    bool useSourceArchive = true; // read source files from source archive library (.sar file)
 
     CppStdRevision cplusplus = CppStdRevision.cpp11;    // version of C++ standard to support
 
@@ -219,6 +221,7 @@ extern (C++) struct Param
     const(char)[] objdir;                // .obj/.lib file output directory
     const(char)[] objname;               // .obj file output name
     const(char)[] libname;               // .lib file output name
+    const(char)[] pathPackage;           // path/package for path/package.sar file
 
     Output ddoc;                        // Generate embedded documentation comments
     Output dihdr;                       // Generate `.di` 'header' files
diff --git a/compiler/src/dmd/globals.h b/compiler/src/dmd/globals.h
index ac2b2867fb7c..063005505842 100644
--- a/compiler/src/dmd/globals.h
+++ b/compiler/src/dmd/globals.h
@@ -119,6 +119,7 @@ struct Help
     d_bool checkAction;  // -checkaction
     d_bool revert;       // -revert
     d_bool preview;      // -preview
+    d_bool sar;          // -sar
     d_bool externStd;    // -extern-std
     d_bool hc;           // -HC
 };
@@ -173,6 +174,8 @@ struct Param
     d_bool addMain;       // add a default main() function
     d_bool allInst;       // generate code for all template instantiations
     d_bool bitfields;         // support C style bit fields
+    d_bool useSourceArchive; // read source files from source archive library (.sar file)
+
     CppStdRevision cplusplus;  // version of C++ name mangling to support
 
     Help help;
@@ -221,6 +224,7 @@ struct Param
     DString objdir;    // .obj/.lib file output directory
     DString objname;   // .obj file output name
     DString libname;   // .lib file output name
+    DString pathPackage; // path to .sar source archive file
 
     Output ddoc;              // Generate embedded documentation comments
     Output dihdr;             // Generate `.di` 'header' files
diff --git a/compiler/src/dmd/main.d b/compiler/src/dmd/main.d
index 27f79379abae..08e5067953df 100644
--- a/compiler/src/dmd/main.d
+++ b/compiler/src/dmd/main.d
@@ -320,6 +320,13 @@ private int tryMain(size_t argc, const(char)** argv, ref Param params)
     {
         fatal();
     }
+    if (params.pathPackage.length)
+    {
+        if (writeSourceArchive(params.pathPackage))
+            fatal();
+        return EXIT_SUCCESS;
+    }
+
     if (files.length == 0)
     {
         if (params.jsonFieldFlags)
@@ -827,6 +834,7 @@ bool parseCommandlineAndConfig(size_t argc, const(char)** argv, ref Param params
               envsection.ptr, cast(int)global.inifilename.length, global.inifilename.ptr);
 
     global.preprocess = &preprocess;
+    global.fileManager.setUseSourceArchive(global.params.useSourceArchive);
     return false;
 }
 
diff --git a/compiler/src/dmd/mars.d b/compiler/src/dmd/mars.d
index 06f694e9e472..363e532a2c4c 100644
--- a/compiler/src/dmd/mars.d
+++ b/compiler/src/dmd/mars.d
@@ -1205,6 +1205,28 @@ bool parseCommandLine(const ref Strings arguments, const size_t argc, ref Param
                 return false;
             }
         }
+        else if (startsWith(p + 1, "sar")) // https://dlang.org/dmd.html#switch-sar
+        {
+            /* Parse:
+             *    -sar=on|off|path/package.sar
+             */
+            enum len = "-sar=".length;
+            mixin(checkOptionsMixin("sar",
+                "`-sar=<behavior>` requires a behavior"));
+            switch (arg[len .. $])
+            {
+            case "on":
+                params.useSourceArchive = true;
+                break;
+            case "off":
+                params.useSourceArchive = false;
+                break;
+            default:
+                params.useSourceArchive = false;
+                params.pathPackage = arg[len .. $];
+                break;
+            }
+        }
         else if (arg == "-w")   // https://dlang.org/dmd.html#switch-w
             params.warnings = DiagnosticReporting.error;
         else if (arg == "-wi")  // https://dlang.org/dmd.html#switch-wi
diff --git a/compiler/src/dmd/root/file.d b/compiler/src/dmd/root/file.d
index 034dba61b13f..ee7170e230c8 100644
--- a/compiler/src/dmd/root/file.d
+++ b/compiler/src/dmd/root/file.d
@@ -92,7 +92,7 @@ nothrow:
 
         version (Posix)
         {
-            //printf("File::read('%s')\n",name);
+            //printf("File::read('%.*s')\n", cast(int)name.length, name.ptr);
             int fd = name.toCStringThen!(slice => open(slice.ptr, O_RDONLY));
             if (fd == -1)
             {
diff --git a/compiler/src/dmd/root/rmem.d b/compiler/src/dmd/root/rmem.d
index 19652072376b..c6986c0b56fe 100644
--- a/compiler/src/dmd/root/rmem.d
+++ b/compiler/src/dmd/root/rmem.d
@@ -318,7 +318,7 @@ Params:
 
 Returns: A null-terminated copy of the input array.
 */
-extern (D) char[] xarraydup(const(char)[] s) pure nothrow
+extern (D) char[] xarraydup(scope const(char)[] s) pure nothrow
 {
     if (!s)
         return null;