cloud-init: 0.7.9 -> 20.2 (python3!)

nixos/modules/system/boot/grow-partition.nix

@@ -23,7 +23,7 @@ with lib;
       copy_bin_and_libs ${pkgs.utillinux}/sbin/sfdisk
       copy_bin_and_libs ${pkgs.utillinux}/sbin/lsblk
 
-      substitute "${pkgs.cloud-utils}/bin/.growpart-wrapped" "$out/bin/growpart" \
+      substitute "${pkgs.cloud-utils.guest}/bin/.growpart-wrapped" "$out/bin/growpart" \
         --replace "${pkgs.bash}/bin/sh" "/bin/sh" \
         --replace "awk" "gawk" \
         --replace "sed" "gnused"

nixos/tests/cloud-init.nix

@@ -7,6 +7,9 @@ with import ../lib/testing-python.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let
+  inherit (import ./ssh-keys.nix pkgs)
+    snakeOilPrivateKey snakeOilPublicKey;
+
   metadataDrive = pkgs.stdenv.mkDerivation {
     name = "metadata";
     buildCommand = ''
@@ -18,13 +21,19 @@ let
       -   content: |
                 cloudinit
           path: /tmp/cloudinit-write-file
+
+      users:
+        - default
+        - name: nixos
+          ssh_authorized_keys:
+            - "${snakeOilPublicKey}"
       EOF
 
       cat << EOF > $out/iso/meta-data
       instance-id: iid-local01
       local-hostname: "test"
       public-keys:
-          - "should be a key!"
+        - "${snakeOilPublicKey}"
       EOF
       ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso
       '';
@@ -34,19 +43,40 @@ in makeTest {
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ lewo ];
   };
-  machine =
-    { ... }:
-    {
-      virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ];
-      services.cloud-init.enable = true;
-    };
+  machine = { ... }:
+  {
+    virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ];
+    services.cloud-init.enable = true;
+    services.openssh.enable = true;
+    networking.hostName = "";
+  };
   testScript = ''
-      machine.start()
-      machine.wait_for_unit("cloud-init.service")
-      machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'")
+    machine.wait_for_unit("cloud-init.service")
+    machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'")
+
+    # install snakeoil ssh key and provision .ssh/config file
+    machine.succeed("mkdir -p ~/.ssh")
+    machine.succeed(
+        "cat ${snakeOilPrivateKey} > ~/.ssh/id_snakeoil"
+    )
+    machine.succeed("chmod 600 ~/.ssh/id_snakeoil")
+
+    machine.wait_for_unit("sshd.service")
+
+    # we should be able to log in as the root user, as well as the created nixos user
+    machine.succeed(
+        "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil root@localhost 'true'"
+    )
+    machine.succeed(
+        "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'true'"
+    )
 
-      machine.wait_until_succeeds(
-          "cat /root/.ssh/authorized_keys | grep -q 'should be a key!'"
-      )
+    # test changing hostname via cloud-init worked
+    assert (
+        machine.succeed(
+            "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'hostname'"
+        ).strip()
+        == "test"
+    )
   '';
 }

pkgs/tools/misc/cloud-utils/default.nix

@@ -1,9 +1,18 @@
 { stdenv, fetchurl, makeWrapper
 , gawk, gnused, utillinux, file
 , wget, python3, qemu-utils, euca2ools
-, e2fsprogs, cdrkit }:
+, e2fsprogs, cdrkit
+, gptfdisk }:
 
-stdenv.mkDerivation rec {
+let
+  # according to https://packages.debian.org/sid/cloud-image-utils + https://packages.debian.org/sid/admin/cloud-guest-utils
+  guestDeps = [
+    e2fsprogs gptfdisk gawk gnused utillinux
+  ];
+  binDeps = guestDeps ++ [
+    wget file qemu-utils cdrkit
+  ];
+in stdenv.mkDerivation rec {
   # NOTICE: if you bump this, make sure to run
   # $ nix-build nixos/release-combined.nix -A nixos.tests.ec2-nixops
   # growpart is needed in initrd in nixos/system/boot/grow-partition.nix
@@ -17,15 +26,24 @@ stdenv.mkDerivation rec {
   buildInputs = [ python3 ];
   installFlags = [ "LIBDIR=$(out)/lib" "BINDIR=$(out)/bin" "MANDIR=$(out)/man/man1" "DOCDIR=$(out)/doc" ];
 
-  # according to https://packages.ubuntu.com/source/zesty/cloud-utils
-  binDeps = [
-    wget e2fsprogs file gnused gawk utillinux qemu-utils euca2ools cdrkit
-  ];
+  # $guest output contains all executables needed for cloud-init and $out the rest + $guest
+  # This is similar to debian's package split into cloud-image-utils and cloud-guest-utils
+  # The reason is to reduce the closure size
+  outputs = [ "out" "guest"];
 
   postFixup = ''
+    moveToOutput bin/ec2metadata $guest
+    moveToOutput bin/growpart $guest
+    moveToOutput bin/vcs-run $guest
+
     for i in $out/bin/*; do
       wrapProgram $i --prefix PATH : "${stdenv.lib.makeBinPath binDeps}:$out/bin"
     done
+
+    for i in $guest/bin/*; do
+      wrapProgram $i --prefix PATH : "${stdenv.lib.makeBinPath guestDeps}:$guest/bin"
+      ln -s $i $out/bin
+    done
   '';
 
   dontBuild = true;

pkgs/tools/virtualization/cloud-init/add-nixos-support.patch → pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch

@@ -1,18 +1,36 @@
-diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/__init__.py cloud-init-0.7.6/cloudinit/distros/__init__.py
---- cloud-init-0.7.6.orig/cloudinit/distros/__init__.py	2014-10-10 15:26:25.000000000 +0000
-+++ cloud-init-0.7.6/cloudinit/distros/__init__.py	2016-06-08 07:51:45.230357099 +0000
-@@ -43,6 +43,7 @@
-     'freebsd': ['freebsd'],
-     'suse': ['sles'],
-     'arch': ['arch'],
+From 269cc4c9558549f340ec186d9246654564b2f633 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
+Date: Tue, 18 Aug 2020 10:22:36 +0100
+Subject: [PATCH] add nixos support
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
+---
+ cloudinit/distros/__init__.py |   1 +
+ cloudinit/distros/nixos.py    | 103 ++++++++++++++++++++++++++++++++++
+ 2 files changed, 104 insertions(+)
+ create mode 100644 cloudinit/distros/nixos.py
+
+diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
+index 2537608f..c533b585 100755
+--- a/cloudinit/distros/__init__.py
++++ b/cloudinit/distros/__init__.py
+@@ -47,6 +47,7 @@ OSFAMILIES = {
+     'gentoo': ['gentoo'],
+     'redhat': ['amazon', 'centos', 'fedora', 'rhel'],
+     'suse': ['opensuse', 'sles'],
 +    'nixos': ['nixos'],
  }
 
  LOG = logging.getLogger(__name__)
-diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/cloudinit/distros/nixos.py
---- cloud-init-0.7.6.orig/cloudinit/distros/nixos.py	1970-01-01 00:00:00.000000000 +0000
-+++ cloud-init-0.7.6/cloudinit/distros/nixos.py	2016-06-08 07:50:58.602616595 +0000
-@@ -0,0 +1,98 @@
+diff --git a/cloudinit/distros/nixos.py b/cloudinit/distros/nixos.py
+new file mode 100644
+index 00000000..d53d2a61
+--- /dev/null
++++ b/cloudinit/distros/nixos.py
+@@ -0,0 +1,103 @@
 +# vi: ts=4 expandtab
 +#
 +#    Copyright (C) 2012 Canonical Ltd.
@@ -35,10 +53,13 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou
 +#    You should have received a copy of the GNU General Public License
 +#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 +
++import os
++
 +from cloudinit import distros
 +from cloudinit import helpers
 +from cloudinit import log as logging
 +from cloudinit import util
++from cloudinit import atomic_helper
 +
 +from cloudinit.distros.parsers.hostname import HostnameConf
 +
@@ -52,6 +73,8 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou
 +        # calls from repeatly happening (when they
 +        # should only happen say once per instance...)
 +        self._runner = helpers.Runners(paths)
++        self.usr_lib_exec = os.path.join(os.path.dirname(__file__),
++                                         "../../../../../libexec")
 +        self.osfamily = 'nixos'
 +
 +    def _select_hostname(self, hostname, fqdn):
@@ -72,7 +95,7 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou
 +        if not conf:
 +            conf = HostnameConf('')
 +        conf.set_hostname(your_hostname)
-+        util.write_file(out_fn, str(conf), 0644)
++        atomic_helper.write_file(out_fn, str(conf).encode("utf-8"))
 +
 +    def _read_system_hostname(self):
 +        sys_hostname = self._read_hostname(self.hostname_conf_fn)
@@ -111,3 +134,6 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou
 +
 +    def update_package_sources(self):
 +        raise NotImplementedError()
+-- 
+2.28.0
+

pkgs/tools/virtualization/cloud-init/default.nix

@@ -1,40 +1,85 @@
-{ lib, pythonPackages, fetchurl, cloud-utils }:
+{ lib
+, fetchFromGitHub
+, buildPythonApplication
+, jinja2
+, oauthlib
+, configobj
+, pyyaml
+, requests
+, jsonschema
+, jsonpatch
+, pytest
+, httpretty
+, dmidecode
+, pytestCheckHook
+, shadow
+, cloud-utils
+, openssh
+}:
 
-let version = "0.7.9";
+let version = "20.3";
 
-in pythonPackages.buildPythonApplication {
+in buildPythonApplication {
   pname = "cloud-init";
   inherit version;
   namePrefix = "";
 
-  src = fetchurl {
-    url = "https://launchpad.net/cloud-init/trunk/${version}/+download/cloud-init-${version}.tar.gz";
-    sha256 = "0wnl76pdcj754pl99wxx76hkir1s61x0bg0lh27sdgdxy45vivbn";
+  src = fetchFromGitHub {
+    owner = "canonical";
+    repo = "cloud-init";
+    rev = version;
+    sha256 = "1fmckxf4q4sxjqs758vw7ca0rnhl9hyq67cqpqzz2v3s1gqzjhm4";
   };
 
-  patches = [ ./add-nixos-support.patch ];
+  patches = [ ./0001-add-nixos-support.patch ];
   prePatch = ''
-    patchShebangs ./tools
+    substituteInPlace setup.py --replace /lib/systemd $out/lib/systemd
+  '';
 
-    substituteInPlace setup.py \
-      --replace /usr $out \
-      --replace /etc $out/etc \
-      --replace /lib/systemd $out/lib/systemd \
-      --replace 'self.init_system = ""' 'self.init_system = "systemd"'
+  postInstall = ''
+    install -D -m755 ./tools/write-ssh-key-fingerprints $out/libexec/write-ssh-key-fingerprints
+    for i in $out/libexec/*; do
+      wrapProgram $i --prefix PATH : "${lib.makeBinPath [ openssh ]}"
+    done
+  '';
 
-    substituteInPlace cloudinit/config/cc_growpart.py \
-      --replace 'util.subp(["growpart"' 'util.subp(["${cloud-utils}/bin/growpart"'
+  propagatedBuildInputs = [
+    jinja2
+    oauthlib
+    configobj
+    pyyaml
+    requests
+    jsonschema
+    jsonpatch
+  ];
 
-    # Argparse is part of python stdlib
-    sed -i s/argparse// requirements.txt
-    '';
+  checkInputs = [
+    pytestCheckHook
+    httpretty
+    dmidecode
+    # needed for tests; at runtime we rather want the setuid wrapper
+    shadow
+  ];
 
-  propagatedBuildInputs = with pythonPackages; [ cheetah jinja2 prettytable
-    oauthlib pyserial configobj pyyaml requests jsonpatch ];
+  makeWrapperArgs = [
+    "--prefix PATH : ${lib.makeBinPath [
+      dmidecode cloud-utils.guest
+    ]}/bin"
+  ];
 
-  checkInputs = with pythonPackages; [ contextlib2 httpretty mock unittest2 ];
+  disabledTests = [
+    # tries to create /var
+    "test_dhclient_run_with_tmpdir"
+    # clears path and fails because mkdir is not found
+    "test_path_env_gets_set_from_main"
+    # tries to read from /etc/ca-certificates.conf while inside the sandbox
+    "test_handler_ca_certs"
+  ];
 
-  doCheck = false;
+  preCheck = ''
+    # TestTempUtils.test_mkdtemp_default_non_root does not like TMPDIR=/build
+    export TMPDIR=/tmp
+  '';
 
   meta = {
     homepage = "https://cloudinit.readthedocs.org";

pkgs/top-level/all-packages.nix

@@ -1557,7 +1557,7 @@ in
 
   clog-cli = callPackage ../development/tools/clog-cli { };
 
-  cloud-init = callPackage ../tools/virtualization/cloud-init { };
+  cloud-init = python3.pkgs.callPackage ../tools/virtualization/cloud-init { };
 
   cloudflared = callPackage ../applications/networking/cloudflared { };