ceph: Fix tests by adding back old required python libs. See NixOS#28…

…1858.

Fixes NixOS#241482.

Also fix test putting cluster in unhealthy `POOL_APP_NOT_ENABLED` state;
this seems to be the default state with Ceph 18.

nixos/tests/ceph-multi-node.nix

@@ -185,6 +185,14 @@ let
     monA.succeed(
         "ceph osd pool create multi-node-test 32 32",
         "ceph osd pool ls | grep 'multi-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable multi-node-test nixos-test",
+
         "ceph osd pool rename multi-node-test multi-node-other-test",
         "ceph osd pool ls | grep 'multi-node-other-test'",
     )

nixos/tests/ceph-single-node-bluestore.nix

@@ -145,6 +145,14 @@ let
     monA.succeed(
         "ceph osd pool create single-node-test 32 32",
         "ceph osd pool ls | grep 'single-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable single-node-test nixos-test",
+
         "ceph osd pool rename single-node-test single-node-other-test",
         "ceph osd pool ls | grep 'single-node-other-test'",
     )

nixos/tests/ceph-single-node.nix

@@ -145,6 +145,14 @@ let
     monA.succeed(
         "ceph osd pool create single-node-test 32 32",
         "ceph osd pool ls | grep 'single-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable single-node-test nixos-test",
+
         "ceph osd pool rename single-node-test single-node-other-test",
         "ceph osd pool ls | grep 'single-node-other-test'",
     )

pkgs/development/python-modules/cryptography/40.nix

@@ -0,0 +1,112 @@
+{ lib
+, stdenv
+, callPackage
+, buildPythonPackage
+, fetchPypi
+, rustPlatform
+, cargo
+, rustc
+, setuptoolsRustBuildHook
+, openssl
+, Security
+, isPyPy
+, cffi
+, pkg-config
+, pytestCheckHook
+, pytest-subtests
+, pythonOlder
+, pretend
+, libiconv
+, libxcrypt
+, iso8601
+, py
+, pytz
+, hypothesis
+}:
+
+let
+  cryptography-vectors = callPackage ./40_vectors.nix { };
+in
+buildPythonPackage rec {
+  pname = "cryptography";
+  version = "40.0.1"; # Also update the hash in vectors.nix
+  format = "setuptools";
+  disabled = pythonOlder "3.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
+  };
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    sourceRoot = "${pname}-${version}/${cargoRoot}";
+    name = "${pname}-${version}";
+    hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU=";
+  };
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace "--benchmark-disable" ""
+  '';
+
+  cargoRoot = "src/rust";
+
+  nativeBuildInputs = [
+    rustPlatform.cargoSetupHook
+    setuptoolsRustBuildHook
+    cargo
+    rustc
+    pkg-config
+  ] ++ lib.optionals (!isPyPy) [
+    cffi
+  ];
+
+  buildInputs = [ openssl ]
+    ++ lib.optionals stdenv.isDarwin [ Security libiconv ]
+    ++ lib.optionals (pythonOlder "3.9") [ libxcrypt ];
+
+  propagatedBuildInputs = lib.optionals (!isPyPy) [
+    cffi
+  ];
+
+  nativeCheckInputs = [
+    cryptography-vectors
+    hypothesis
+    iso8601
+    pretend
+    py
+    pytestCheckHook
+    pytest-subtests
+    pytz
+  ];
+
+  pytestFlagsArray = [
+    "--disable-pytest-warnings"
+  ];
+
+  disabledTestPaths = [
+    # save compute time by not running benchmarks
+    "tests/bench"
+  ] ++ lib.optionals (stdenv.isDarwin && stdenv.isAarch64) [
+    # aarch64-darwin forbids W+X memory, but this tests depends on it:
+    # * https://cffi.readthedocs.io/en/latest/using.html#callbacks
+    "tests/hazmat/backends/test_openssl_memleak.py"
+  ];
+
+  meta = with lib; {
+    description = "A package which provides cryptographic recipes and primitives";
+    longDescription = ''
+      Cryptography includes both high level recipes and low level interfaces to
+      common cryptographic algorithms such as symmetric ciphers, message
+      digests, and key derivation functions.
+      Our goal is for it to be your "cryptographic standard library". It
+      supports Python 2.7, Python 3.5+, and PyPy 5.4+.
+    '';
+    homepage = "https://github.com/pyca/cryptography";
+    changelog = "https://cryptography.io/en/latest/changelog/#v"
+      + replaceStrings [ "." ] [ "-" ] version;
+    license = with licenses; [ asl20 bsd3 psfl ];
+    maintainers = with maintainers; [ SuperSandro2000 ];
+  };
+}

pkgs/development/python-modules/cryptography/40_vectors.nix

@@ -0,0 +1,27 @@
+{ buildPythonPackage, fetchPypi, lib, cryptography_40 }:
+
+buildPythonPackage rec {
+  pname = "cryptography-vectors";
+  # The test vectors must have the same version as the cryptography package
+  inherit (cryptography_40) version;
+  format = "setuptools";
+
+  src = fetchPypi {
+    pname = "cryptography_vectors";
+    inherit version;
+    hash = "sha256-hGBwa1tdDOSoVXHKM4nPiPcAu2oMYTPcn+D1ovW9oEE=";
+  };
+
+  # No tests included
+  doCheck = false;
+
+  pythonImportsCheck = [ "cryptography_vectors" ];
+
+  meta = with lib; {
+    description = "Test vectors for the cryptography package";
+    homepage = "https://cryptography.io/en/latest/development/test-vectors/";
+    # Source: https://github.com/pyca/cryptography/tree/master/vectors;
+    license = with licenses; [ asl20 bsd3 ];
+    maintainers = with maintainers; [ SuperSandro2000 ];
+  };
+}

pkgs/development/python-modules/kubernetes/18.nix

@@ -0,0 +1,63 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pythonOlder
+
+# propgatedBuildInputs
+, adal
+, certifi
+, google-auth
+, python-dateutil
+, pyyaml
+, requests
+, requests_oauthlib
+, urllib3
+, websocket-client
+
+# tests
+, pytestCheckHook
+, mock
+}:
+
+buildPythonPackage rec {
+  pname = "kubernetes";
+  version = "18.20.0";
+  format = "setuptools";
+  disabled = pythonOlder "3.6";
+
+  src = fetchFromGitHub {
+    owner = "kubernetes-client";
+    repo = "python";
+    rev = "v${version}";
+    sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
+    fetchSubmodules = true;
+  };
+
+  propagatedBuildInputs = [
+    adal
+    certifi
+    google-auth
+    python-dateutil
+    pyyaml
+    requests
+    requests_oauthlib
+    urllib3
+    websocket-client
+  ];
+
+  pythonImportsCheck = [
+    "kubernetes"
+  ];
+
+  checkInputs = [
+    mock
+    pytestCheckHook
+  ];
+
+  meta = with lib; {
+    description = "Kubernetes python client";
+    homepage = "https://github.com/kubernetes-client/python";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ lsix ];
+  };
+}

pkgs/development/python-modules/pyopenssl/23_1.nix

@@ -0,0 +1,100 @@
+{ lib
+, stdenv
+, buildPythonPackage
+, fetchPypi
+, openssl
+, cryptography
+, pytestCheckHook
+, pretend
+, sphinxHook
+, sphinx-rtd-theme
+, flaky
+}:
+
+buildPythonPackage rec {
+  pname = "pyopenssl";
+  version = "23.1.1";
+  format = "setuptools";
+
+  src = fetchPypi {
+    pname = "pyOpenSSL";
+    inherit version;
+    hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
+  };
+
+  outputs = [
+    "out"
+    "dev"
+    "doc"
+  ];
+
+  nativeBuildInputs = [
+    openssl
+    sphinxHook
+    sphinx-rtd-theme
+  ];
+
+  postPatch = ''
+    # remove cryptography pin
+    sed "/cryptography/ s/,<[0-9]*//g" setup.py
+  '';
+
+  propagatedBuildInputs = [
+    cryptography
+  ];
+
+  nativeCheckInputs = [
+    flaky
+    pretend
+    pytestCheckHook
+  ];
+
+  __darwinAllowLocalNetworking = true;
+
+  preCheck = ''
+    export LANG="en_US.UTF-8"
+  '';
+
+  disabledTests = [
+    # https://github.com/pyca/pyopenssl/issues/692
+    # These tests, we disable always.
+    "test_set_default_verify_paths"
+    "test_fallback_default_verify_paths"
+    # https://github.com/pyca/pyopenssl/issues/768
+    "test_wantWriteError"
+    # https://github.com/pyca/pyopenssl/issues/1043
+    "test_alpn_call_failure"
+  ] ++ lib.optionals (lib.hasPrefix "libressl" openssl.meta.name) [
+    # https://github.com/pyca/pyopenssl/issues/791
+    # These tests, we disable in the case that libressl is passed in as openssl.
+    "test_op_no_compression"
+    "test_npn_advertise_error"
+    "test_npn_select_error"
+    "test_npn_client_fail"
+    "test_npn_success"
+    "test_use_certificate_chain_file_unicode"
+    "test_use_certificate_chain_file_bytes"
+    "test_add_extra_chain_cert"
+    "test_set_session_id_fail"
+    "test_verify_with_revoked"
+    "test_set_notAfter"
+    "test_set_notBefore"
+  ] ++ lib.optionals (lib.versionAtLeast (lib.getVersion openssl.name) "1.1") [
+    # these tests are extremely tightly wed to the exact output of the openssl cli tool, including exact punctuation.
+    "test_dump_certificate"
+    "test_dump_privatekey_text"
+    "test_dump_certificate_request"
+    "test_export_text"
+  ] ++ lib.optionals stdenv.is32bit [
+    # https://github.com/pyca/pyopenssl/issues/974
+    "test_verify_with_time"
+  ];
+
+  meta = with lib; {
+    description = "Python wrapper around the OpenSSL library";
+    homepage = "https://github.com/pyca/pyopenssl";
+    changelog = "https://github.com/pyca/pyopenssl/blob/${version}/CHANGELOG.rst";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ];
+  };
+}

pkgs/tools/filesystems/ceph/default.nix

@@ -167,6 +167,18 @@ let
   # Watch out for python <> boost compatibility
   python = python310.override {
     packageOverrides = self: super: {
+      # Ceph does not support `cryptography` > 40 yet:
+      #     https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
+      # We pin the older `cryptography_40` here;
+      # this also forces us to pin an older `pyopenssl` because the current one
+      # is not compatible with older `cryptography`, see:
+      #     https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
+      cryptography = super.cryptography_40;
+      pyopenssl = super.pyopenssl_23_1;
+
+      # Ceph does not support `kubernetes` >= 19, see:
+      #     https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
+      kubernetes = super.kubernetes_18;
     };
   };