Merge staging-next into staging

doc/languages-frameworks/python.section.md

@@ -833,6 +833,7 @@ used in `buildPythonPackage`.
 - `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder.
 - `setuptoolsBuildHook` to build a wheel using `setuptools`.
 - `setuptoolsCheckHook` to run tests with `python setup.py test`.
+- `venvShellHook` to source a Python 3 `venv` at the `venvDir` location. A `venv` is created if it does not yet exist.
 - `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`.
 
 ### Development mode

doc/languages-frameworks/rust.section.md

@@ -32,17 +32,17 @@ Rust applications are packaged by using the `buildRustPackage` helper from `rust
 
 ```
 rustPlatform.buildRustPackage rec {
-  name = "ripgrep-${version}";
-  version = "0.4.0";
+  pname = "ripgrep";
+  version = "11.0.2";
 
   src = fetchFromGitHub {
     owner = "BurntSushi";
-    repo = "ripgrep";
-    rev = "${version}";
-    sha256 = "0y5d1n6hkw85jb3rblcxqas2fp82h3nghssa4xqrhqnz25l799pj";
+    repo = pname;
+    rev = version;
+    sha256 = "1iga3320mgi7m853la55xip514a3chqsdi1a1rwv25lr9b1p7vd3";
   };
 
-  cargoSha256 = "0q68qyl2h6i0qsz82z840myxlnjay8p1w5z7hfyr8fqp7wgwa9cx";
+  cargoSha256 = "17ldqr3asrdcsh4l29m3b5r37r5d0b3npq1lrgjmxb6vlx6a36qh";
   verifyCargoDeps = true;
 
   meta = with stdenv.lib; {
@@ -66,7 +66,11 @@ added in `cargoPatches` will also be prepended to the patches in `patches` at
 build-time.
 
 When `verifyCargoDeps` is set to `true`, the build will also verify that the
-`cargoSha256` is not out of date by comparing the `Cargo.lock` file in both the `cargoDeps` and `src`. Note that this option changes the value of `cargoSha256` since it also copies the `Cargo.lock` in it. To avoid breaking backward-compatibility this option is not enabled by default but hopefully will be in the future.
+`cargoSha256` is not out of date by comparing the `Cargo.lock` file in both the
+`cargoDeps` and `src`. Note that this option changes the value of `cargoSha256`
+since it also copies the `Cargo.lock` in it. To avoid breaking
+backward-compatibility this option is not enabled by default but hopefully will
+be in the future.
 
 ### Building a crate for a different target
 

doc/overrides.css

@@ -1,9 +1,22 @@
 .docbook .xref img[src^=images\/callouts\/],
 .screen img,
-.programlisting img {
+.programlisting img,
+.literallayout img,
+.synopsis img {
     width: 1em;
 }
 
 .calloutlist img {
     width: 1.5em;
 }
+
+.prompt,
+.screen img,
+.programlisting img,
+.literallayout img,
+.synopsis img {
+    -moz-user-select: none;
+    -webkit-user-select: none;
+    -ms-user-select: none;
+    user-select: none;
+}

lib/cli.nix

@@ -0,0 +1,56 @@
+{ lib }:
+
+rec {
+  /* Automatically convert an attribute set to command-line options.
+
+     This helps protect against malformed command lines and also to reduce
+     boilerplate related to command-line construction for simple use cases.
+
+     Example:
+       encodeGNUCommandLine
+         { }
+         { data = builtins.toJSON { id = 0; };
+
+           X = "PUT";
+
+           retry = 3;
+
+           retry-delay = null;
+
+           url = [ "https://example.com/foo" "https://example.com/bar" ];
+
+           silent = false;
+
+           verbose = true;
+         };
+       => "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"
+  */
+  encodeGNUCommandLine =
+    options: attrs: lib.escapeShellArgs (toGNUCommandLine options attrs);
+
+  toGNUCommandLine =
+    { renderKey ?
+        key: if builtins.stringLength key == 1 then "-${key}" else "--${key}"
+
+    , renderOption ?
+        key: value:
+          if value == null
+          then []
+          else [ (renderKey key) (builtins.toString value) ]
+
+    , renderBool ? key: value: lib.optional value (renderKey key)
+
+    , renderList ? key: value: lib.concatMap (renderOption key) value
+    }:
+    options:
+      let
+        render = key: value:
+                 if builtins.isBool value
+            then renderBool key value
+            else if builtins.isList value
+            then renderList key value
+            else renderOption key value;
+
+      in
+        builtins.concatLists (lib.mapAttrsToList render options);
+}

lib/default.nix

@@ -39,6 +39,7 @@ let
 
     # misc
     asserts = callLibs ./asserts.nix;
+    cli = callLibs ./cli.nix;
     debug = callLibs ./debug.nix;
     generators = callLibs ./generators.nix;
     misc = callLibs ./deprecated.nix;
@@ -100,7 +101,7 @@ let
     inherit (sources) pathType pathIsDirectory cleanSourceFilter
       cleanSource sourceByRegex sourceFilesBySuffices
       commitIdFromGitRepo cleanSourceWith pathHasContext
-      canCleanSource;
+      canCleanSource pathIsRegularFile;
     inherit (modules) evalModules unifyModuleSyntax
       applyIfFunction mergeModules
       mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions
@@ -120,6 +121,7 @@ let
       isOptionType mkOptionType;
     inherit (asserts)
       assertMsg assertOneOf;
+    inherit (cli) encodeGNUCommandLine toGNUCommandLine;
     inherit (debug) addErrorContextToAttrs traceIf traceVal traceValFn
       traceXMLVal traceXMLValMarked traceSeq traceSeqN traceValSeq
       traceValSeqFn traceValSeqN traceValSeqNFn traceShowVal

lib/sources.nix

@@ -9,6 +9,9 @@ rec {
   # Returns true if the path exists and is a directory, false otherwise
   pathIsDirectory = p: if builtins.pathExists p then (pathType p) == "directory" else false;
 
+  # Returns true if the path exists and is a regular file, false otherwise
+  pathIsRegularFile = p: if builtins.pathExists p then (pathType p) == "regular" else false;
+
   # Bring in a path as a source, filtering out all Subversion and CVS
   # directories, as well as backup files (*~).
   cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
@@ -110,24 +113,43 @@ rec {
       with builtins;
         let fileName       = toString path + "/" + file;
             packedRefsName = toString path + "/packed-refs";
-        in if lib.pathExists fileName
+        in if pathIsRegularFile path
+           # Resolve git worktrees. See gitrepository-layout(5)
+           then
+             let m   = match "^gitdir: (.*)$" (lib.fileContents path);
+             in if m == null
+                then throw ("File contains no gitdir reference: " + path)
+                else
+                  let gitDir     = lib.head m;
+                      commonDir' = if pathIsRegularFile "${gitDir}/commondir"
+                                   then lib.fileContents "${gitDir}/commondir"
+                                   else gitDir;
+                      commonDir  = if lib.hasPrefix "/" commonDir'
+                                   then commonDir'
+                                   else toString (/. + "${gitDir}/${commonDir'}");
+                      refFile    = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
+                  in readCommitFromFile refFile commonDir
+
+           else if pathIsRegularFile fileName
+           # Sometimes git stores the commitId directly in the file but
+           # sometimes it stores something like: «ref: refs/heads/branch-name»
            then
              let fileContent = lib.fileContents fileName;
-                 # Sometimes git stores the commitId directly in the file but
-                 # sometimes it stores something like: «ref: refs/heads/branch-name»
                  matchRef    = match "^ref: (.*)$" fileContent;
-             in if   matchRef == null
+             in if  matchRef == null
                 then fileContent
                 else readCommitFromFile (lib.head matchRef) path
+
+           else if pathIsRegularFile packedRefsName
            # Sometimes, the file isn't there at all and has been packed away in the
            # packed-refs file, so we have to grep through it:
-           else if lib.pathExists packedRefsName
            then
              let fileContent = readFile packedRefsName;
                  matchRef    = match (".*\n([^\n ]*) " + file + "\n.*") fileContent;
-             in if   matchRef == null
+             in if  matchRef == null
                 then throw ("Could not find " + file + " in " + packedRefsName)
                 else lib.head matchRef
+
            else throw ("Not a .git directory: " + path);
     in readCommitFromFile "HEAD";
 

lib/tests/misc.nix

@@ -441,4 +441,25 @@ runTests {
     expected  = "«foo»";
   };
 
+  testRenderOptions = {
+    expr =
+       encodeGNUCommandLine
+         { }
+         { data = builtins.toJSON { id = 0; };
+
+           X = "PUT";
+
+           retry = 3;
+
+           retry-delay = null;
+
+           url = [ "https://example.com/foo" "https://example.com/bar" ];
+
+           silent = false;
+
+           verbose = true;
+         };
+
+    expected = "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'";
+  };
 }

lib/tests/modules.sh

@@ -174,8 +174,7 @@ checkConfigOutput "true" config.submodule.inner ./declare-submoduleWith-modules.
 checkConfigOutput "true" config.submodule.outer ./declare-submoduleWith-modules.nix
 
 ## Paths should be allowed as values and work as expected
-# Temporarily disabled until https://github.com/NixOS/nixpkgs/pull/76861
-#checkConfigOutput "true" config.submodule.enable ./declare-submoduleWith-path.nix
+checkConfigOutput "true" config.submodule.enable ./declare-submoduleWith-path.nix
 
 # Check that disabledModules works recursively and correctly
 checkConfigOutput "true" config.enable ./disable-recursive/main.nix

lib/trivial.nix

@@ -191,7 +191,7 @@ rec {
     let
       revisionFile = "${toString ./..}/.git-revision";
       gitRepo      = "${toString ./..}/.git";
-    in if lib.pathIsDirectory gitRepo
+    in if builtins.pathExists gitRepo
        then lib.commitIdFromGitRepo gitRepo
        else if lib.pathExists revisionFile then lib.fileContents revisionFile
        else default;

lib/types.nix

@@ -340,29 +340,79 @@ rec {
             let
               padWidth = stringLength (toString (length def.value));
               unnamed = i: unnamedPrefix + fixedWidthNumber padWidth i;
+              anyString = placeholder "name";
+              nameAttrs = [
+                { path = [ "environment" "etc" ];
+                  name = "target";
+                }
+                { path = [ "containers" anyString "bindMounts" ];
+                  name = "mountPoint";
+                }
+                { path = [ "programs" "ssh" "knownHosts" ];
+                  # hostNames is actually a list so we would need to handle it only when singleton
+                  name = "hostNames";
+                }
+                { path = [ "fileSystems" ];
+                  name = "mountPoint";
+                }
+                { path = [ "boot" "specialFileSystems" ];
+                  name = "mountPoint";
+                }
+                { path = [ "services" "znapzend" "zetup" ];
+                  name = "dataset";
+                }
+                { path = [ "services" "znapzend" "zetup" anyString "destinations" ];
+                  name = "label";
+                }
+                { path = [ "services" "geoclue2" "appConfig" ];
+                  name = "desktopID";
+                }
+              ];
+              matched = let
+                equals = a: b: b == anyString || a == b;
+                fallback = { name = "name"; };
+              in findFirst ({ path, ... }: all (v: v == true) (zipListsWith equals loc path)) fallback nameAttrs;
+              nameAttr = matched.name;
+              nameValueOld = value:
+                if isList value then
+                  if length value > 0 then
+                    "[ " + concatMapStringsSep " " escapeNixString value + " ]"
+                  else
+                    "[ ]"
+                else
+                  escapeNixString value;
+              nameValueNew = value: unnamed:
+                if isList value then
+                  if length value > 0 then
+                    head value
+                  else
+                    unnamed
+                else
+                  value;
               res =
                 { inherit (def) file;
                   value = listToAttrs (
                     imap1 (elemIdx: elem:
-                      { name  = elem.name or (unnamed elemIdx);
+                      { name  = nameValueNew (elem.${nameAttr} or (unnamed elemIdx)) (unnamed elemIdx);
                         value = elem;
                       }) def.value);
                 };
               option = concatStringsSep "." loc;
               sample = take 3 def.value;
-              list = concatMapStrings (x: ''{ name = "${x.name or "unnamed"}"; ...} '') sample;
-              set = concatMapStrings (x: ''${x.name or "unnamed"} = {...}; '') sample;
+              more = lib.optionalString (length def.value > 3) "... ";
+              list = concatMapStrings (x: ''{ ${nameAttr} = ${nameValueOld (x.${nameAttr} or "unnamed")}; ...} '') sample;
+              set = concatMapStrings (x: ''${nameValueNew (x.${nameAttr} or "unnamed") "unnamed"} = {...}; '') sample;
               msg = ''
                 In file ${def.file}
                 a list is being assigned to the option config.${option}.
                 This will soon be an error as type loaOf is deprecated.
                 See https://git.io/fj2zm for more information.
                 Do
                   ${option} =
-                    { ${set}...}
+                    { ${set}${more}}
                 instead of
                   ${option} =
-                    [ ${list}...]
+                    [ ${list}${more}]
               '';
             in
               lib.warn msg res
@@ -430,14 +480,16 @@ rec {
           else unify (if shorthandOnlyDefinesConfig then { config = value; } else value);
 
         allModules = defs: modules ++ imap1 (n: { value, file }:
-          # Annotate the value with the location of its definition for better error messages
-          coerce (lib.modules.unifyModuleSyntax file "${toString file}-${toString n}") value
+          if isAttrs value || isFunction value then
+            # Annotate the value with the location of its definition for better error messages
+            coerce (lib.modules.unifyModuleSyntax file "${toString file}-${toString n}") value
+          else value
         ) defs;
 
       in
       mkOptionType rec {
         name = "submodule";
-        check = x: isAttrs x || isFunction x;
+        check = x: isAttrs x || isFunction x || path.check x;
         merge = loc: defs:
           (evalModules {
             modules = allModules defs;
@@ -538,7 +590,7 @@ rec {
         tail' = tail ts;
       in foldl' either head' tail';
 
-    # Either value of type `finalType` or `coercedType`, the latter is
+    # Either value of type `coercedType` or `finalType`, the former is
     # converted to `finalType` using `coerceFunc`.
     coercedTo = coercedType: coerceFunc: finalType:
       assert lib.assertMsg (coercedType.getSubModules == null)
@@ -547,12 +599,12 @@ rec {
       mkOptionType rec {
         name = "coercedTo";
         description = "${finalType.description} or ${coercedType.description} convertible to it";
-        check = x: finalType.check x || (coercedType.check x && finalType.check (coerceFunc x));
+        check = x: (coercedType.check x && finalType.check (coerceFunc x)) || finalType.check x;
         merge = loc: defs:
           let
             coerceVal = val:
-              if finalType.check val then val
-              else coerceFunc val;
+              if coercedType.check val then coerceFunc val
+              else val;
           in finalType.merge loc (map (def: def // { value = coerceVal def.value; }) defs);
         emptyValue = finalType.emptyValue;
         getSubOptions = finalType.getSubOptions;