Fix 3.1.0 upgrade problems, restore and enhance serialization prefs

docs/docs.md

@@ -147,11 +147,13 @@ saving the module to the database (e.g., as `/db/test.xqm`) and selecting "XQuer
 
 ## Options for XML documents loaded from the database
 
-When loading XML documents from the database into an editor window, you can control whether indentation is automatically applied 
-or not and whether XInclude elements are automatically expanded or not. Set this preference in "Edit > Preferences > When 
-opening or downloading XML documents." The same preference applies to the download of XML documents via "File > Download" and
+When loading XML documents from the database into an editor window, you can control common serialization parameters, such 
+as whether indentation is automatically applied, whether XInclude elements are automatically expanded, and whether XML 
+declarations are omitted. Set these preferences in "Edit > Preferences > Serialization." Additional sets of these preferences 
+are available for download of XML documents via "File > Download" and for
 the serialization of XML documents included in application packages via ["Application > Download app"](#Support-for-EXPath-Packages).
-By default, indentation is turned on and XInclude expansion is turned off.
+By default, indentation is turned on for opening and downloading files but is off for downloading EXPath Packages; XInclude 
+expansion is turned off in all cases; and the XML declaration is omitted in all cases. 
 
 ## Options for displaying query results
 

index.html.tmpl

@@ -454,7 +454,8 @@
                           to the selected collection. Maximum file size is 100MB.</p>
                                 <form class="file_upload" action="modules/upload.xql" method="POST" enctype="multipart/form-data">
                                     <div class="options">
-                                        <input type="checkbox" name="deploy"/> Auto deploy uploaded .xar packages
+                                        <input type="checkbox" name="deploy" id="auto-deploy"/> 
+                                        <label for="auto-deploy">Auto deploy uploaded .xar packages</label>
                                     </div>
                                     <div class="file_upload_drop">Drop files here ...</div>
                                     <div class="file_upload_body">
@@ -581,6 +582,7 @@
                 <div id="preferences-dialog">
                     <form>
                         <fieldset>
+                            <legend>Editor</legend>
                             <ol>
                                 <li>
                                     <select id="theme" name="theme">
@@ -646,7 +648,7 @@
                                     <label for="font-size">Font Size:</label>
                                 </li>
                                 <li>
-                                    <input type="checkbox" name="print-margin" checked="checked"/>
+                                    <input type="checkbox" name="print-margin"/>
                                     <label for="print-margin">Show Print Margin:</label>
                                 </li>
                                 <li>
@@ -662,7 +664,7 @@
                                         <option value="4">4 spaces</option>
                                         <option value="8">8 spaces</option>
                                     </select>
-                                    <label for="indent-size">indent size:</label>
+                                    <label for="indent-size">Indent size:</label>
                                 </li>
                                 <li>
                                     <select name="soft-wrap">
@@ -683,6 +685,58 @@
                                 </li>
                             </ol>
                         </fieldset>
+                        <fieldset>
+                            <legend>Serialization</legend>
+                            <p>Control common serialization parameters when opening or downloading
+                                resources and EXPath packages: (a) whether indentation is applied to
+                                XML documents, (b) whether XInclude elements are expanded, and (c)
+                                whether the XML declaration should be omitted.</p>
+                            <h3>Opening files (via File > Open)</h3>
+                            <ol>
+                                <li>
+                                    <label for="indent-on-open">Indent:</label>
+                                    <input type="checkbox" name="indent-on-open" id="indent-on-open"/>
+                                </li>
+                                <li>
+                                    <label for="expand-xincludes-on-open">Expand XIncludes:</label>
+                                    <input type="checkbox" name="expand-xincludes-on-open" id="expand-xincludes-on-open"/>
+                                </li>
+                                <li>
+                                    <label for="omit-xml-decl-on-open">Omit XML Declaration:</label>
+                                    <input type="checkbox" name="omit-xml-decl-on-open" id="omit-xml-decl-on-open"/>
+                                </li>
+                            </ol>
+                            <h3>Downloading files (via File > Download)</h3>
+                            <ol>
+                                <li>
+                                    <label for="indent-on-download">Indent:</label>
+                                    <input type="checkbox" name="indent-on-download" id="indent-on-download"/>
+                                </li>
+                                <li>
+                                    <label for="expand-xincludes-on-download">Expand XIncludes:</label>
+                                    <input type="checkbox" name="expand-xincludes-on-download" id="expand-xincludes-on-download"/>
+                                </li>
+                                <li>
+                                    <label for="omit-xml-decl-on-download">Omit XML Declaration:</label>
+                                    <input type="checkbox" name="omit-xml-decl-on-download" id="omit-xml-decl-on-download"/>
+                                </li>
+                            </ol>
+                            <h3>Downloading EXPath packages (via Application > Download App)</h3>
+                            <ol>
+                                <li>
+                                    <label for="indent-on-download-package">Indent:</label>
+                                    <input type="checkbox" name="indent-on-download-package" id="indent-on-download-package"/>
+                                </li>
+                                <li>
+                                    <label for="expand-xincludes-on-download-package">Expand XIncludes:</label>
+                                    <input type="checkbox" name="expand-xincludes-on-download-package" id="expand-xincludes-on-download-package"/>
+                                </li>
+                                <li>
+                                    <label for="omit-xml-decl-on-download-package">Omit XML Declaration:</label>
+                                    <input type="checkbox" name="omit-xml-decl-on-download-package" id="omit-xml-decl-on-download-package"/>
+                                </li>
+                            </ol>
+                        </fieldset>
                     </form>
                 </div>
                 <div id="select-module-dialog">

modules/dbutils.xql

@@ -0,0 +1,62 @@
+xquery version "3.0";
+
+module namespace dbutil="http://exist-db.org/xquery/dbutil";
+
+import module namespace sm="http://exist-db.org/xquery/securitymanager";
+import module namespace xmldb="http://exist-db.org/xquery/xmldb";
+
+(:~ Scan a collection tree recursively starting at $root. Call $func once for each collection found :)
+declare function dbutil:scan-collections($root as xs:anyURI, $func as function(xs:anyURI) as item()*) {
+    $func($root),
+    if (sm:has-access($root, "rx")) then
+        for $child in xmldb:get-child-collections($root)
+        return
+            dbutil:scan-collections(xs:anyURI($root || "/" || $child), $func)
+    else
+        ()
+};
+
+(:~
+ : List all resources contained in a collection and call the supplied function once for each
+ : resource with the complete path to the resource as parameter.
+ :)
+declare function dbutil:scan-resources($collection as xs:anyURI, $func as function(xs:anyURI) as item()*) {
+    if (sm:has-access($collection, "rx")) then
+        for $child in xmldb:get-child-resources($collection)
+        return
+            $func(xs:anyURI($collection || "/" || $child))
+    else
+        ()
+};
+
+(:~ 
+ : Scan a collection tree recursively starting at $root. Call the supplied function once for each
+ : resource encountered. The first parameter to $func is the collection URI, the second the resource
+ : path (including the collection part).
+ :)
+declare function dbutil:scan($root as xs:anyURI, $func as function(xs:anyURI, xs:anyURI?) as item()*) {
+    dbutil:scan-collections($root, function($collection as xs:anyURI) {
+        $func($collection, ()),
+        (:  scan-resources expects a function with one parameter, so we use a partial application
+            to fill in the collection parameter :)
+        dbutil:scan-resources($collection, $func($collection, ?))
+    })
+};
+
+declare function dbutil:find-by-mimetype($collection as xs:anyURI, $mimeType as xs:string+) {
+    dbutil:scan($collection, function($collection, $resource) {
+        if (exists($resource) and xmldb:get-mime-type($resource) = $mimeType) then
+            $resource
+        else
+            ()
+    })
+};
+
+declare function dbutil:find-by-mimetype($collection as xs:anyURI, $mimeType as xs:string+, $func as function(xs:anyURI) as item()*) {
+    dbutil:scan($collection, function($collection, $resource) {
+        if (exists($resource) and xmldb:get-mime-type($resource) = $mimeType) then
+            $func($resource)
+        else
+            ()
+    })
+};

modules/deployment.xql

@@ -20,14 +20,13 @@ xquery version "3.1";
 
 import module namespace apputil="http://exist-db.org/apps/eXide/apputil" at "util.xql";
 import module namespace tmpl="http://exist-db.org/xquery/template" at "tmpl.xql";
-import module namespace dbutil="http://exist-db.org/xquery/dbutil";
+import module namespace dbutil="http://exist-db.org/xquery/dbutil" at "dbutils.xql";
 
 (:~
     Edit the expath and repo app descriptors.
     Functions to read, update the descriptors and deploy an app.
 :)
 declare namespace deploy="http://exist-db.org/eXide/deploy";
-declare namespace git="http://exist-db.org/eXide/git";
 declare namespace expath="http://expath.org/ns/pkg";
 declare namespace repo="http://exist-db.org/xquery/repo";
 
@@ -586,7 +585,7 @@ declare function deploy:package($collection as xs:string, $expathConf as element
         xmldb:store("/db/system/repo", $name, $xar, "application/zip")
 };
 
-declare function deploy:download($app-collection as xs:string, $expathConf as element(), $expand-xincludes as xs:boolean, $indent as xs:boolean) {
+declare function deploy:download($app-collection as xs:string, $expathConf as element(), $expand-xincludes as xs:boolean, $indent as xs:boolean, $omit-xml-declaration as xs:boolean) {
     let $name := concat($expathConf/@abbrev, "-", $expathConf/@version, ".xar")
     let $entries :=
         (: compression:zip uses default serialization parameters, so we'll construct entries manually :)
@@ -606,6 +605,8 @@ declare function deploy:download($app-collection as xs:string, $expathConf as el
                                 || (if ($expand-xincludes) then "yes" else "no")
                                 || " indent=" 
                                 || (if ($indent) then "yes" else "no")
+                                || " omit-xml-declaration=" 
+                                || (if ($omit-xml-declaration) then "yes" else "no")
                             ),
                             doc($resource)
                         }</entry>
@@ -641,15 +642,16 @@ let $collection :=
         repo:get-root() || $target
 let $info := request:get-parameter("info", ())
 let $download := request:get-parameter("download", ())
-let $expand-xincludes := request:get-parameter("expand-xincludes", "false") cast as xs:boolean
-let $indent := request:get-parameter("indent", "false") cast as xs:boolean
+let $expand-xincludes := request:get-parameter("expand-xincludes", false()) cast as xs:boolean
+let $indent := request:get-parameter("indent", false()) cast as xs:boolean
+let $omit-xml-declaration := request:get-parameter("omit-xml-decl", true()) cast as xs:boolean
 let $expathConf := if ($collection) then xmldb:xcollection($collection)/expath:package else ()
 let $repoConf := if ($collection) then xmldb:xcollection($collection)/repo:meta else ()
 let $abbrev := request:get-parameter("abbrev", ())
 return
     try {
         if ($download) then
-            deploy:download($collection, $expathConf, $expand-xincludes, $indent)
+            deploy:download($collection, $expathConf, $expand-xincludes, $indent, $omit-xml-declaration)
         else if ($info) then
             apputil:get-info($info)
         else if ($abbrev) then

modules/load.xql

@@ -20,8 +20,6 @@ xquery version "3.0";
 
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
 
-declare option exist:serialize "indent=yes expand-xincludes=no";
-
 declare function local:get-run-path($path) {
     let $appRoot := repo:get-root()
     return
@@ -37,6 +35,9 @@ declare function local:get-run-path($path) {
 
 let $path := request:get-parameter("path", ())
 let $download := request:get-parameter("download", ())
+let $indent := request:get-parameter("indent", true()) cast as xs:boolean
+let $expand-xincludes := request:get-parameter("expand-xincludes", false()) cast as xs:boolean
+let $omit-xml-declaration := request:get-parameter("omit-xml-decl", true()) cast as xs:boolean
 let $mime := xmldb:get-mime-type($path)
 let $isBinary := util:is-binary-doc($path)
 (: Disable betterFORM filter :)
@@ -55,11 +56,21 @@ return
             return
                 response:stream-binary($data, $mime, ())
         else
-            let $doc := doc($path)
-            return
-                if ($doc) then
-                    $doc
-                else
-                    response:set-status-code(404)
+            if (doc-available($path)) then
+                (
+                    (: workaround until https://github.com/eXist-db/exist/issues/2394 is resolved :)
+                    util:declare-option(
+                        "exist:serialize", 
+                        "indent=" 
+                            || (if ($indent) then "yes" else "no")
+                            || " expand-xincludes=" 
+                            || (if ($expand-xincludes) then "yes" else "no")
+                            || " omit-xml-declaration=" 
+                            || (if ($omit-xml-declaration) then "yes" else "no")
+                    ),
+                    doc($path)
+                )
+            else
+                response:set-status-code(404)
     else
         response:set-status-code(404)

modules/search.xql

@@ -1,7 +1,7 @@
 xquery version "3.1";
 
 
-import module namespace dbutil="http://exist-db.org/xquery/dbutil";
+import module namespace dbutil="http://exist-db.org/xquery/dbutil" at "dbutils.xql";
 
 declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
 

resources/css/eXide.css

@@ -1156,6 +1156,14 @@ fieldset legend {
     margin-bottom: 8px;
 }
 
+#preferences-dialog h3 {
+    padding-left: 2em;
+}
+
+#preferences-dialog p {
+    padding-left: 1em;
+}
+
 #dialog-deploy legend {
     background: url("../images/bubble.gif") top right no-repeat;
 }

src/deployment.js

@@ -212,7 +212,10 @@ eXide.edit.PackageEditor = (function () {
     eXide.util.oop.inherit(Constr, eXide.events.Sender);
 
     Constr.prototype.download = function (collection) {
-        window.location.href = "modules/deployment.xql?download=true&collection=" + encodeURIComponent(collection);
+        var indentOnDownloadPackage = $("#indent-on-download-package").is(":checked");
+        var expandXIncludesOnDownloadPackage = $("#expand-xincludes-on-download-package").is(":checked");
+        var omitXMLDeclatarionOnDownloadPackage = $("#omit-xml-decl-on-download-package").is(":checked");
+        window.location.href = "modules/deployment.xql?download=true&collection=" + encodeURIComponent(collection) + "&indent=" + indentOnDownloadPackage + "&expand-xincludes=" + expandXIncludesOnDownloadPackage + "&omit-xml-decl=" + omitXMLDeclatarionOnDownloadPackage;
     };
 
 	/**

src/eXide.js

@@ -291,6 +291,9 @@ eXide.app = (function(util) {
 
 		$doOpenDocument: function(resource, callback, reload) {
 			resource.path = util.normalizePath(resource.path);
+            var indentOnOpen = $("#indent-on-open").is(":checked");
+            var expandXIncludesOnOpen = $("#expand-xincludes-on-open").is(":checked");
+            var omitXMLDeclatarionOnOpen = $("#omit-xml-decl-on-open").is(":checked");
             var doc = editor.getDocument(resource.path);
             if (doc && !reload) {
                 editor.switchTo(doc);
@@ -300,8 +303,9 @@ eXide.app = (function(util) {
                 return true;
             }
 			$.ajax({
-				url: "modules/load.xql?path=" + resource.path,
+				url: "modules/load.xql",
 				dataType: 'text',
+                data: { "path": resource.path, "indent": indentOnOpen, "expand-xincludes": expandXIncludesOnOpen, "omit-xml-decl": omitXMLDeclatarionOnOpen },
 				success: function (data, status, xhr) {
                     if (reload) {
                         editor.reload(data);
@@ -446,12 +450,15 @@ eXide.app = (function(util) {
         },
 
 		download: function() {
+            var indentOnDownload = $("#indent-on-download").is(":checked");
+            var expandXIncludesOnDownload = $("#expand-xincludes-on-download").is(":checked");
+            var omitXMLDeclatarionOnDownload = $("#omit-xml-decl-on-download").is(":checked");
 			var doc = editor.getActiveDocument();
 			if (doc.getPath().match("^__new__") || !doc.isSaved()) {
 				util.error("There are unsaved changes in the document. Please save it first.");
 				return;
 			}
-			window.location.href = "modules/load.xql?download=true&path=" + encodeURIComponent(doc.getPath());
+            window.location.href = "modules/load.xql?download=true&path=" + encodeURIComponent(doc.getPath()) + "&indent=" + indentOnDownload + "&expand-xincludes=" + expandXIncludesOnDownload + "&omit-xml-decl=" + omitXMLDeclatarionOnDownload;
 		},
 
 		runQuery: function(path, livePreview) {