v18.2.3

PendingReleaseNotes

@@ -100,6 +100,13 @@
    `ceph config get mgr mgr/volumes/snapshot_clone_no_wait`
   and it can be disabled by using:
    `ceph config set mgr mgr/volumes/snapshot_clone_no_wait false`
+* CephFS: fixes to the implementation of the ``root_squash`` mechanism enabled
+  via cephx ``mds`` caps on a client credential require a new client feature
+  bit, ``client_mds_auth_caps``. Clients using credentials with ``root_squash``
+  without this feature will trigger the MDS to raise a HEALTH_ERR on the
+  cluster, MDS_CLIENTS_BROKEN_ROOTSQUASH. See the documentation on this warning
+  and the new feature bit for more information.
+
 
 >=18.0.0
 

ceph.spec.in

@@ -2042,6 +2042,7 @@ fi
 
 %files -n ceph-exporter
 %{_bindir}/ceph-exporter
+%{_unitdir}/ceph-exporter.service
 
 %files -n rbd-fuse
 %{_bindir}/rbd-fuse
@@ -2634,6 +2635,5 @@ exit 0
 %dir %{python3_sitelib}/ceph_node_proxy
 %{python3_sitelib}/ceph_node_proxy/*
 %{python3_sitelib}/ceph_node_proxy-*
-#%{_mandir}/man8/ceph-node-proxy.8*
 
 %changelog

cmake/modules/BuildArrow.cmake

@@ -86,6 +86,9 @@ function(build_arrow)
   else()
     list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
   endif()
+  # don't add -Werror or debug package builds fail with:
+  #warning _FORTIFY_SOURCE requires compiling with optimization (-O)
+  list(APPEND arrow_CMAKE_ARGS -DBUILD_WARNING_LEVEL=PRODUCTION)
 
   # we use an external project and copy the sources to bin directory to ensure
   # that object files are built outside of the source tree.

debian/ceph-exporter.install

@@ -0,0 +1,2 @@
+lib/systemd/system/ceph-exporter*
+usr/bin/ceph-exporter

debian/ceph-mgr.requires

@@ -1,3 +1,4 @@
+bcrypt
 pyOpenSSL
 cephfs
 ceph-argparse

debian/control

@@ -352,6 +352,30 @@ Description: debugging symbols for ceph-mgr
  .
  This package contains the debugging symbols for ceph-mgr.
 
+Package: ceph-exporter
+Architecture: linux-any
+Depends: ceph-base (= ${binary:Version}),
+Description: metrics exporter for the ceph distributed storage system
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the metrics exporter daemon, which is used to expose
+ the performance metrics.
+
+Package: ceph-exporter-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph-exporter (= ${binary:Version}),
+         ${misc:Depends},
+Description: debugging symbols for ceph-exporter
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the debugging symbols for ceph-exporter.
+
 Package: ceph-mon
 Architecture: linux-any
 Depends: ceph-base (= ${binary:Version}),

debian/rules

@@ -105,6 +105,7 @@ override_dh_strip:
 	dh_strip -pceph-mds --dbg-package=ceph-mds-dbg
 	dh_strip -pceph-fuse --dbg-package=ceph-fuse-dbg
 	dh_strip -pceph-mgr --dbg-package=ceph-mgr-dbg
+	dh_strip -pceph-exporter --dbg-package=ceph-exporter-dbg
 	dh_strip -pceph-mon --dbg-package=ceph-mon-dbg
 	dh_strip -pceph-osd --dbg-package=ceph-osd-dbg
 	dh_strip -pceph-base --dbg-package=ceph-base-dbg

doc/cephfs/administration.rst

@@ -258,31 +258,47 @@ Clients that are missing newly added features will be evicted automatically.
 
 Here are the current CephFS features and first release they came out:
 
-+------------------+--------------+-----------------+
-| Feature          | Ceph release | Upstream Kernel |
-+==================+==============+=================+
-| jewel            | jewel        | 4.5             |
-+------------------+--------------+-----------------+
-| kraken           | kraken       | 4.13            |
-+------------------+--------------+-----------------+
-| luminous         | luminous     | 4.13            |
-+------------------+--------------+-----------------+
-| mimic            | mimic        | 4.19            |
-+------------------+--------------+-----------------+
-| reply_encoding   | nautilus     | 5.1             |
-+------------------+--------------+-----------------+
-| reclaim_client   | nautilus     | N/A             |
-+------------------+--------------+-----------------+
-| lazy_caps_wanted | nautilus     | 5.1             |
-+------------------+--------------+-----------------+
-| multi_reconnect  | nautilus     | 5.1             |
-+------------------+--------------+-----------------+
-| deleg_ino        | octopus      | 5.6             |
-+------------------+--------------+-----------------+
-| metric_collect   | pacific      | N/A             |
-+------------------+--------------+-----------------+
-| alternate_name   | pacific      | PLANNED         |
-+------------------+--------------+-----------------+
++----------------------------+--------------+-----------------+
+| Feature                    | Ceph release | Upstream Kernel |
++============================+==============+=================+
+| jewel                      | jewel        | 4.5             |
++----------------------------+--------------+-----------------+
+| kraken                     | kraken       | 4.13            |
++----------------------------+--------------+-----------------+
+| luminous                   | luminous     | 4.13            |
++----------------------------+--------------+-----------------+
+| mimic                      | mimic        | 4.19            |
++----------------------------+--------------+-----------------+
+| reply_encoding             | nautilus     | 5.1             |
++----------------------------+--------------+-----------------+
+| reclaim_client             | nautilus     | N/A             |
++----------------------------+--------------+-----------------+
+| lazy_caps_wanted           | nautilus     | 5.1             |
++----------------------------+--------------+-----------------+
+| multi_reconnect            | nautilus     | 5.1             |
++----------------------------+--------------+-----------------+
+| deleg_ino                  | octopus      | 5.6             |
++----------------------------+--------------+-----------------+
+| metric_collect             | pacific      | N/A             |
++----------------------------+--------------+-----------------+
+| alternate_name             | pacific      | 6.5             |
++----------------------------+--------------+-----------------+
+| notify_session_state       | quincy       | 5.19            |
++----------------------------+--------------+-----------------+
+| op_getvxattr               | quincy       | 6.0             |
++----------------------------+--------------+-----------------+
+| 32bits_retry_fwd           | reef         | 6.6             |
++----------------------------+--------------+-----------------+
+| new_snaprealm_info         | reef         | UNKNOWN         |
++----------------------------+--------------+-----------------+
+| has_owner_uidgid           | reef         | 6.6             |
++----------------------------+--------------+-----------------+
+| client_mds_auth_caps       | squid+bp     | PLANNED         |
++----------------------------+--------------+-----------------+
+
+..
+    Comment: use `git describe --tags --abbrev=0 <commit>` to lookup release
+
 
 CephFS Feature Descriptions
 
@@ -340,6 +356,15 @@ Clients can send performance metric to MDS if MDS support this feature.
 Clients can set and understand "alternate names" for directory entries. This is
 to be used for encrypted file name support.
 
+::
+
+    client_mds_auth_caps
+
+To effectively implement ``root_squash`` in a client's ``mds`` caps, the client
+must understand that it is enforcing ``root_squash`` and other cap metadata.
+Clients without this feature are in danger of dropping updates to files.  It is
+recommend to set this feature bit.
+
 
 Global settings
 ---------------

doc/cephfs/health-messages.rst

@@ -252,3 +252,20 @@ other daemons, please see :ref:`health-checks`.
     dirty data for cap revokes). If ``defer_client_eviction_on_laggy_osds`` is
     set to true (default true), client eviction will not take place and thus
     this health warning will be generated.
+
+``MDS_CLIENTS_BROKEN_ROOTSQUASH``
+---------------------------------
+  Message
+    "X client(s) with broken root_squash implementation (MDS_CLIENTS_BROKEN_ROOTSQUASH)"
+
+  Description
+    A bug was discovered in root_squash which would potentially lose changes made by a
+    client restricted with root_squash caps. The fix required a change to the protocol
+    and a client upgrade is required.
+
+    This is a HEALTH_ERR warning because of the danger of inconsistency and lost
+    data. It is recommended to either upgrade your clients, discontinue using
+    root_squash in the interim, or silence the warning if desired.
+
+    To evict and permanently block broken clients from connecting to the
+    cluster, set the ``required_client_feature`` bit ``client_mds_auth_caps``.

qa/tasks/cephfs/mount.py

@@ -551,30 +551,21 @@ def _verify_attrs(self, **kwargs):
                 raise RuntimeError('value of attributes should be either str '
                                    f'or None. {k} - {v}')
 
-    def update_attrs(self, client_id=None, client_keyring_path=None,
-                     client_remote=None, hostfs_mntpt=None, cephfs_name=None,
-                     cephfs_mntpt=None):
-        if not (client_id or client_keyring_path or client_remote or
-                cephfs_name or cephfs_mntpt or hostfs_mntpt):
-            return
-
-        self._verify_attrs(client_id=client_id,
-                           client_keyring_path=client_keyring_path,
-                           hostfs_mntpt=hostfs_mntpt, cephfs_name=cephfs_name,
-                           cephfs_mntpt=cephfs_mntpt)
-
-        if client_id:
-            self.client_id = client_id
-        if client_keyring_path:
-            self.client_keyring_path = client_keyring_path
-        if client_remote:
-            self.client_remote = client_remote
-        if hostfs_mntpt:
-            self.hostfs_mntpt = hostfs_mntpt
-        if cephfs_name:
-            self.cephfs_name = cephfs_name
-        if cephfs_mntpt:
-            self.cephfs_mntpt = cephfs_mntpt
+    def update_attrs(self, **kwargs):
+        verify_keys = [
+          'client_id',
+          'client_keyring_path',
+          'hostfs_mntpt',
+          'cephfs_name',
+          'cephfs_mntpt',
+        ]
+
+        self._verify_attrs(**{key: kwargs[key] for key in verify_keys if key in kwargs})
+
+        for k in verify_keys:
+            v = kwargs.get(k)
+            if v is not None:
+                setattr(self, k, v)
 
     def remount(self, **kwargs):
         """
@@ -597,7 +588,7 @@ def remount(self, **kwargs):
 
         self.update_attrs(**kwargs)
 
-        retval = self.mount(mntopts=mntopts, check_status=check_status)
+        retval = self.mount(mntopts=mntopts, check_status=check_status, **kwargs)
         # avoid this scenario (again): mount command might've failed and
         # check_status might have silenced the exception, yet we attempt to
         # wait which might lead to an error.

qa/tasks/cephfs/test_admin.py

@@ -1319,6 +1319,92 @@ def test_single_path_rootsquash(self):
         self.captester.conduct_neg_test_for_chown_caps()
         self.captester.conduct_neg_test_for_truncate_caps()
 
+    def test_multifs_rootsquash_nofeature(self):
+        """
+        That having root_squash on one fs doesn't prevent access to others.
+        """
+
+        if not isinstance(self.mount_a, FuseMount):
+            self.skipTest("only FUSE client has CEPHFS_FEATURE_MDS_AUTH_CAPS "
+                          "needed to enforce root_squash MDS caps")
+
+        self.fs1 = self.fs
+        self.fs2 = self.mds_cluster.newfs('testcephfs2')
+
+        self.mount_a.umount_wait()
+
+        self.run_ceph_cmd(f'auth caps client.{self.mount_a.client_id} '
+                          f'mon "allow r" '
+                          f'osd "allow rw tag cephfs data={self.fs1.name}, allow rw tag cephfs data={self.fs2.name}" '
+                          f'mds "allow rwp fsname={self.fs1.name}, allow rw fsname={self.fs2.name} root_squash"')
+
+        CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK = 21
+        # all but CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK
+        features = ",".join([str(i) for i in range(CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK)])
+        mntargs = [f"--client_debug_inject_features={features}"]
+
+        # should succeed
+        with self.assert_cluster_log("report clients with broken root_squash", present=False):
+            self.mount_a.remount(mntargs=mntargs, cephfs_name=self.fs1.name)
+
+    def test_rootsquash_nofeature(self):
+        """
+        That having root_squash on an fs without the feature bit raises a HEALTH_ERR warning.
+        """
+
+        if not isinstance(self.mount_a, FuseMount):
+            self.skipTest("only FUSE client has CEPHFS_FEATURE_MDS_AUTH_CAPS "
+                          "needed to enforce root_squash MDS caps")
+
+        self.mount_a.umount_wait()
+
+        FS_AUTH_CAPS = (('/', 'rw', 'root_squash'),)
+        keyring = self.fs.authorize(self.client_id, FS_AUTH_CAPS)
+
+        CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK = 21
+        # all but CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK
+        features = ",".join([str(i) for i in range(CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK)])
+        mntargs = [f"--client_debug_inject_features={features}"]
+
+        # should succeed
+        with self.assert_cluster_log("with broken root_squash implementation"):
+            keyring_path = self.mount_a.client_remote.mktemp(data=keyring)
+            self.mount_a.remount(client_id=self.client_id, client_keyring_path=keyring_path, mntargs=mntargs, cephfs_name=self.fs.name)
+            self.wait_for_health("MDS_CLIENTS_BROKEN_ROOTSQUASH", 60)
+            self.assertFalse(self.mount_a.is_blocked())
+
+        self.mount_a.umount_wait()
+        self.wait_for_health_clear(60)
+
+    def test_rootsquash_nofeature_evict(self):
+        """
+        That having root_squash on an fs without the feature bit can be evicted.
+        """
+
+        if not isinstance(self.mount_a, FuseMount):
+            self.skipTest("only FUSE client has CEPHFS_FEATURE_MDS_AUTH_CAPS "
+                          "needed to enforce root_squash MDS caps")
+
+        self.mount_a.umount_wait()
+
+        FS_AUTH_CAPS = (('/', 'rw', 'root_squash'),)
+        keyring = self.fs.authorize(self.client_id, FS_AUTH_CAPS)
+
+        CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK = 21
+        # all but CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK
+        features = ",".join([str(i) for i in range(CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK)])
+        mntargs = [f"--client_debug_inject_features={features}"]
+
+        # should succeed
+        keyring_path = self.mount_a.client_remote.mktemp(data=keyring)
+        self.mount_a.remount(client_id=self.client_id, client_keyring_path=keyring_path, mntargs=mntargs, cephfs_name=self.fs.name)
+        self.wait_for_health("MDS_CLIENTS_BROKEN_ROOTSQUASH", 60)
+
+        self.fs.required_client_features("add", "client_mds_auth_caps")
+        self.wait_for_health_clear(60)
+        self.assertTrue(self.mount_a.is_blocked())
+
+
     def test_single_path_rootsquash_issue_56067(self):
         """
         That a FS client using root squash MDS caps allows non-root user to write data

src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test.yml

@@ -14,29 +14,29 @@
   tasks:
 
     - name: mark osds down
-      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      command: "ceph osd down osd.{{ item }}"
       with_items: "{{ osd_ids }}"
     - name: purge osds
-      command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it"
+      command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it"
       with_items: "{{ osd_ids }}"
 
 - hosts: osds
   become: yes
   tasks:
 
     - name: zap devices used for OSDs
-      command: "ceph-volume --cluster {{ cluster }} lvm zap {{ item }} --destroy"
+      command: "ceph-volume lvm zap {{ item }} --destroy"
       with_items: "{{ devices }}"
       environment:
         CEPH_VOLUME_DEBUG: 1
 
     - name: batch create devices again
-      command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
+      command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
       environment:
         CEPH_VOLUME_DEBUG: 1
 
     - name: ensure batch create is idempotent
-      command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
+      command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
       register: batch_cmd
       failed_when: false
       environment:
@@ -50,7 +50,7 @@
          - "'strategy changed' not in batch_cmd.stderr"
 
     - name: run batch --report to see if devices get filtered
-      command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
+      command: "ceph-volume lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}"
       register: report_cmd
       failed_when: false
       environment: