[WIP] mgr/dashboard_v2: Add RGW user and bucket lists (read-only)

ceph.spec.in

@@ -221,12 +221,23 @@ BuildRequires:	python3-Cython
 BuildRequires:	python%{_python_buildid}-cherrypy
 BuildRequires:	python%{_python_buildid}-pecan
 BuildRequires:	python%{_python_buildid}-werkzeug
+BuildRequires:	python%{_python_buildid}-tox
+BuildRequires:	python%{_python_buildid}-coverage
+%if 0%{?fedora}
+BuildRequires:	python%{_python_buildid}-bcrypt
+%endif
+%if 0%{?rhel}
+BuildRequires:	py-bcrypt
+%endif
 %endif
 %if 0%{?suse_version}
 BuildRequires:	python%{_python_buildid}-CherryPy
 BuildRequires:	python%{_python_buildid}-Werkzeug
 BuildRequires:	python%{_python_buildid}-pecan
 BuildRequires:	python%{_python_buildid}-numpy-devel
+BuildRequires:	python%{_python_buildid}-bcrypt
+BuildRequires:	python%{_python_buildid}-tox
+BuildRequires:	python%{_python_buildid}-coverage
 %endif
 %endif
 # lttng and babeltrace for rbd-replay-prep
@@ -358,13 +369,20 @@ Requires:       python%{_python_buildid}-jinja2
 Requires:       python%{_python_buildid}-pecan
 Requires:       python%{_python_buildid}-werkzeug
 Requires:       pyOpenSSL%{_python_buildid}
+%if 0%{?fedora}
+Requires:	python%{_python_buildid}-bcrypt
+%endif
+%if 0%{?rhel}
+Requires:	py-bcrypt
+%endif
 %endif
 %if 0%{?suse_version}
 Requires:       python%{_python_buildid}-CherryPy
 Requires:       python%{_python_buildid}-Jinja2
 Requires:       python%{_python_buildid}-Werkzeug
 Requires:       python%{_python_buildid}-pecan
 Requires:       python%{_python_buildid}-pyOpenSSL
+Requires:       python%{_python_buildid}-bcrypt
 Recommends:     python%{_python_buildid}-influxdb
 %endif
 %description mgr
@@ -888,6 +906,7 @@ cmake .. \
     -DWITH_EMBEDDED=OFF \
     -DWITH_MANPAGE=ON \
     -DWITH_PYTHON3=ON \
+    -DWITH_MGR_DASHBOARD_V2_FRONTEND=OFF \
 %if %{with python2}
     -DWITH_PYTHON2=ON \
 %else

debian/control

@@ -54,6 +54,9 @@ Build-Depends: bc,
                python-jinja2,
                python-nose,
                python-pecan,
+               python-bcrypt,
+               python-tox,
+               python-coverage,
                python-prettytable,
                python-setuptools,
                python-sphinx,
@@ -173,6 +176,7 @@ Depends: ceph-base (= ${binary:Version}),
          python-cherrypy3,
          python-openssl,
          python-pecan,
+         python-bcrypt,
          python-werkzeug,
          ${misc:Depends},
          ${python:Depends},

debian/rules

@@ -5,7 +5,7 @@ export DESTDIR=$(CURDIR)/debian/tmp
 
 export DEB_HOST_ARCH      ?= $(shell dpkg-architecture -qDEB_HOST_ARCH)
 
-extraopts += -DUSE_CRYPTOPP=OFF -DWITH_OCF=ON -DWITH_LTTNG=ON -DWITH_PYTHON3=ON -DWITH_EMBEDDED=OFF
+extraopts += -DUSE_CRYPTOPP=OFF -DWITH_OCF=ON -DWITH_LTTNG=ON -DWITH_PYTHON3=ON -DWITH_EMBEDDED=OFF -DWITH_MGR_DASHBOARD_V2_FRONTEND=OFF
 extraopts += -DWITH_CEPHFS_JAVA=ON
 extraopts += -DWITH_SYSTEMD=ON -DCEPH_SYSTEMD_ENV_DIR=/etc/default
 # assumes that ceph is exmpt from multiarch support, so we override the libdir.

doc/mgr/dashboard.rst

@@ -2,7 +2,29 @@ dashboard plugin
 ================
 
 Dashboard plugin visualizes the statistics of the cluster using a web server
-hosted by ``ceph-mgr``.
+hosted by ``ceph-mgr``. The dashboard currently provides insight into the
+following aspects of your Ceph cluster:
+
+* **Overall cluster health**: The overall cluster status, storage utilization
+  (e.g. number of objects, raw capacity, usage per pool), a list of pools and
+  their status and usage statistics, access to the cluster log file.
+* **Hosts**: A list of all hosts associated to the cluster, which services
+  are running and which version of Ceph is installed.
+* **Performance counters**: Display detailed statistics for each running service.
+* **Monitors**: List of all MONs, their quorum status, open sessions.
+* **Configuration Reference**: List all available configuration options, their
+  description and default values.
+* **OSDs**: A list of all OSDs, their status and usage statistics as well as
+  detailed information like attributes (OSD map), metadata, performance counters
+  and usage histograms for read/write operations.
+* **iSCSI**: List all hosts that run the TCMU runner service, list of all images
+  and their performance characteristics (read/write ops, traffic).
+* **RBD**: List of all RBD images and their properties (size, objects, features)
+  in a given pool. 
+* **CephFS**: List all active filesystem clients and associated pools, including
+  their usage statistics.
+* **Object Gateway**: List of all active object gateways and their performance
+  counters.
 
 Enabling
 --------
@@ -39,6 +61,23 @@ If the port is not configured, the web app will bind to port ``7000``.
 If the address it not configured, the web app will bind to ``::``,
 which corresponds to all available IPv4 and IPv6 addresses.
 
+In order to be able to log in, you need to define a username and password, which
+will be stored in the MON's configuration database::
+
+  ceph dashboard set-login-credentials <username> <password>
+
+The password will be stored in the configuration database in encrypted form
+using ``bcrypt``. This is a global setting that applies to all dashboard instances.
+
+You can now access the dashboard using your (JavaScript-enabled) web browser, by
+pointing it to the selected TCP port and any of the host names or IP addresses
+where a manager instance runs on, e.g. ``http://<$IP>:<$PORT>/``.
+
+You should then be greeted by the dashboard login page, requesting your
+previously defined username and password. Select the **Keep me logged in**
+checkbox if you want to skip the username/password request when accessing the
+dashboard in the future.
+
 Reverse proxies
 ---------------
 
@@ -51,5 +90,4 @@ to use hyperlinks that include your prefix, you can set the
 
   ceph config-key set mgr/dashboard/url_prefix $PREFIX
 
-so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.
-
+so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.

install-deps.sh

@@ -105,6 +105,72 @@ EOF
     fi
 }
 
+function ensure_min_npm_version {
+  local install_npm_pkg_cmd=$1
+
+  if ! type npm > /dev/null 2>&1; then
+    $SUDO $install_npm_pkg_cmd
+  fi
+
+  NODE_VER=`node -v`
+  NODE_VER_MAJOR=`node -v | sed 's/v\(\w\+\).*/\1/g'`
+  NODE_VER_MINOR=`node -v | sed 's/v\w\+\.\(\w\+\).*/\1/g'`
+
+  # The minimum node version required is 4.8.0 so that we can use yarn below
+  UPDATE_NODE=false
+  if [ $NODE_VER_MAJOR -lt 4 ]; then
+    UPDATE_NODE=true
+  elif [ $NODE_VER_MAJOR -eq 4 ] && [ $NODE_VER_MINOR -lt 8 ]; then
+    UPDATE_NODE=true
+  fi
+  if $UPDATE_NODE; then
+    ARCH=`uname -m`
+    if [ "$ARCH" = "x86_64" ]; then
+      ARCH="x64"
+    elif [ "$ARCH" = "aarch64" ]; then
+      ARCH="arm64"
+    fi
+    $SUDO npm install -g n
+    # installs nodejs version 4.8.0
+    # we need to explicitly pass the ARCH type because of a bug in the
+    # n package that fails to detect the system architecture in aarch64
+    # systems
+    $SUDO n --arch $ARCH 4.8.0
+    $SUDO npm uninstall -g n
+    hash -d node > /dev/null 2>&1 || true
+  fi
+
+  NPM_VER=`npm -v`
+  NPM_VER_MAJOR=`npm -v | sed 's/\(\w\+\).*/\1/g'`
+
+  # The minimum npm version required is 5.0.0 so that we can install and use
+  # a local nodejs installation (required by the dashboard angular2 frontend)
+  if [ $NPM_VER_MAJOR -lt 5 ]; then
+    $SUDO npm install -g yarn
+    $SUDO yarn global add npm@^5.0.0  # install npm version 5.0.0 or later
+  fi
+  hash -d npm > /dev/null 2>&1 || true
+
+  NEW_NODE_VER=`node -v`
+  NEW_NPM_VER=`npm -v`
+  if [ ! "$NODE_VER" = "$NEW_NODE_VER" ]; then
+cat <<EOF
+*****************************************************************************
+      YOUR NODE VERSION WAS UPDATED FROM $NODE_VER TO $NEW_NODE_VER
+*****************************************************************************
+EOF
+  fi
+  if [ ! "$NPM_VER" = "$NEW_NPM_VER" ]; then
+cat <<EOF
+*****************************************************************************
+      YOUR NPM VERSION WAS UPDATED FROM $NPM_VER TO $NEW_NPM_VER
+      TO RETURN TO VERSION $NPM_VER run the following command:
+          $ $SUDO yarn global remove npm && hash -d npm
+*****************************************************************************
+EOF
+  fi
+}
+
 if [ x`uname`x = xFreeBSDx ]; then
     $SUDO pkg install -yq \
         devel/babeltrace \
@@ -190,6 +256,9 @@ else
 	$SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
 	$SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
 	if [ -n "$backports" ] ; then rm $control; fi
+        $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y install nodejs
+        [ ! -e /usr/bin/node ] && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y install nodejs-legacy
+        ensure_min_npm_version "env DEBIAN_FRONTEND=noninteractive apt-get -y install npm"
         ;;
     centos|fedora|rhel|ol|virtuozzo)
         yumdnf="yum"
@@ -250,13 +319,15 @@ else
             ensure_decent_gcc_on_rh $dts_ver
 	fi
         ! grep -q -i error: $DIR/yum-builddep.out || exit 1
+        ensure_min_npm_version "$yumdnf install -y npm"
         ;;
     opensuse|suse|sles)
         echo "Using zypper to install dependencies"
         zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
         $SUDO $zypp_install lsb-release systemd-rpm-macros
         munge_ceph_spec_in $DIR/ceph.spec
         $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
+        ensure_min_npm_version "zypper --non-interactive install npm"
         ;;
     alpine)
         # for now we need the testing repo for leveldb

make-dist

@@ -61,6 +61,15 @@ download_boost() {
     rm -rf src/boost
 }
 
+build_dashboard_frontend() {
+  CURR_DIR=`pwd`
+  cd src/pybind/mgr/dashboard_v2/frontend
+  npm install
+  npm run build -- --prod
+  cd $CURR_DIR
+  tar cf dashboard_frontend.tar $outfile/src/pybind/mgr/dashboard_v2/frontend/dist
+}
+
 # clean out old cruft...
 echo "cleanup..."
 rm -f $outfile*
@@ -113,9 +122,11 @@ download_boost $boost_version b2dfbd6c717be4a7bb2d88018eaccf75 \
                https://dl.bintray.com/boostorg/release/$boost_version/source \
                https://downloads.sourceforge.net/project/boost/boost/$boost_version \
                https://download.ceph.com/qa
+build_dashboard_frontend
 tar --concatenate -f $outfile.all.tar $outfile.version.tar
 tar --concatenate -f $outfile.all.tar $outfile.boost.tar
 tar --concatenate -f $outfile.all.tar $outfile.tar
+tar --concatenate -f $outfile.all.tar dashboard_frontend.tar
 mv $outfile.all.tar $outfile.tar
 rm $outfile
 rm -f $outfile.version.tar

qa/suites/rados/mgr/tasks/dashboard_v2.yaml

@@ -0,0 +1,33 @@
+
+tasks:
+  - install:
+  - ceph:
+      # tests may leave mgrs broken, so don't try and call into them
+      # to invoke e.g. pg dump during teardown.
+      wait-for-scrub: false
+      log-whitelist:
+        - overall HEALTH_
+        - \(MGR_DOWN\)
+        - \(PG_
+        - replacing it with standby
+        - No standby daemons available
+        - \(FS_DEGRADED\)
+        - \(MDS_FAILED\)
+        - \(MDS_DEGRADED\)
+        - \(FS_WITH_FAILED_MDS\)
+        - \(MDS_DAMAGE\)
+  - rgw: [client.0]
+  - cephfs_test_runner:
+      modules:
+        - tasks.mgr.test_dashboard_v2
+        - tasks.mgr.dashboard_v2.test_auth
+        - tasks.mgr.dashboard_v2.test_cephfs
+        - tasks.mgr.dashboard_v2.test_cluster_configuration
+        - tasks.mgr.dashboard_v2.test_dashboard
+        - tasks.mgr.dashboard_v2.test_host
+        - tasks.mgr.dashboard_v2.test_monitor
+        - tasks.mgr.dashboard_v2.test_osd
+        - tasks.mgr.dashboard_v2.test_perf_counters
+        - tasks.mgr.dashboard_v2.test_summary
+        - tasks.mgr.dashboard_v2.test_rgw
+        - tasks.mgr.dashboard_v2.test_rbd

qa/tasks/ceph_test_case.py

@@ -132,7 +132,8 @@ def wait_until_equal(self, get_fn, expect_val, timeout, reject_fn=None):
 
         log.debug("wait_until_equal: success")
 
-    def wait_until_true(self, condition, timeout):
+    @classmethod
+    def wait_until_true(cls, condition, timeout):
         period = 5
         elapsed = 0
         while True: