mgr/dashboard_v2: New Web-UI (v1 feature parity)

Merge pull request ceph#20103 from openattic/wip-mgr-dashboard_v2 Reviewed-by: Nathan Cutler <ncutler@suse.com> Reviewed-by: John Spray <john.spray@redhat.com> Reviewed-by: Kefu Chai <kchai@redhat.com> (cherry picked from commit 23eb14d) Signed-off-by: Ernesto Puerta <epuertat@redhat.com> Conflicts: ceph.spec.in install-deps.sh qa/tasks/mgr/mgr_test_case.py qa/tasks/mgr/test_prometheus.py src/pybind/CMakeLists.txt src/vstart.sh Conflict-solving summary: ceph.spec.in: - Add make-check distro-conditional Python BuildRequires: CherryPy, tox, coverage and bcrypt. - Add ceph-mgr package Python Requires: bcrypt install-deps.sh - Add ensure_min_npm_version qa/tasks/mgr/mgr_test_case.py - Pick setUpClass method from remote qa/tasks/mgr/test_prometheus.py - Delete src/pybind/CMakeLists.txt - Add add_subdirectory(mgr) src/vstart.sh - Bring DASH_V2_URLS and dashboard_v2 config setting.
rhcs-dashboard · Aug 23, 2018 · 186d3f2 · 186d3f2
1 parent 1c0f777
commit 186d3f2
Show file tree

Hide file tree

Showing 289 changed files with 13,322 additions and 48 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -586,6 +586,9 @@ else()
 endif()
 set(Boost_USE_MULTITHREADED ON)
 
+# dashboard angular2 frontend
+option(WITH_MGR_DASHBOARD_V2_FRONTEND "Build the mgr/dashboard_v2 frontend using `npm`" ON)
+
 include_directories(SYSTEM ${PROJECT_BINARY_DIR}/include)
 
 find_package(Threads REQUIRED)

diff --git a/ceph.spec.in b/ceph.spec.in
@@ -200,6 +200,26 @@ BuildRequires:	python3-devel
 BuildRequires:	python3-setuptools
 BuildRequires:	python3-Cython
 %endif
+# distro-conditional make check dependencies
+%if 0%{with make_check}
+%if 0%{?fedora} || 0%{?rhel}
+BuildRequires:	python-cherrypy
+BuildRequires:	python-tox
+BuildRequires:	python-coverage
+%if 0%{?fedora}
+BuildRequires:	python-bcrypt
+%endif
+%if 0%{?rhel}
+BuildRequires:	py-bcrypt
+%endif
+%endif
+%if 0%{?suse_version}
+BuildRequires:	python-CherryPy
+BuildRequires:	python-bcrypt
+BuildRequires:	python-tox
+BuildRequires:	python-coverage
+%endif
+%endif
 # lttng and babeltrace for rbd-replay-prep
 %if %{with lttng}
 %if 0%{?fedora} || 0%{?rhel}
@@ -329,12 +349,19 @@ Requires:       python-cherrypy
 Requires:       python-jinja2
 Requires:       python-werkzeug
 Requires:       pyOpenSSL
+%if 0%{?fedora}
+Requires:	python-bcrypt
+%endif
+%if 0%{?rhel}
+Requires:	py-bcrypt
+%endif
 %endif
 %if 0%{?suse_version}
 Requires: 	python-CherryPy
 Requires:       python-Jinja2
 Requires:       python-Werkzeug
 Requires:       python-pyOpenSSL
+Requires:       python-bcrypt
 %endif
 Requires:       python-pecan
 %description mgr
@@ -826,6 +853,7 @@ cmake .. \
     -DCMAKE_INSTALL_INCLUDEDIR=%{_includedir} \
     -DWITH_MANPAGE=ON \
     -DWITH_PYTHON3=ON \
+    -DWITH_MGR_DASHBOARD_V2_FRONTEND=OFF \
     -DWITH_SYSTEMD=ON \
 %if 0%{?rhel} && ! 0%{?centos}
     -DWITH_SUBMAN=ON \

diff --git a/debian/control b/debian/control
@@ -53,6 +53,9 @@ Build-Depends: bc,
                python-cherrypy3,
                python-nose,
                python-pecan,
+               python-bcrypt,
+               python-tox,
+               python-coverage,
                python-prettytable,
                python-setuptools,
                python-six,
@@ -173,6 +176,7 @@ Depends: ceph-base (= ${binary:Version}),
          python-jinja2,
          python-openssl,
          python-pecan,
+         python-bcrypt,
          python-werkzeug,
          ${misc:Depends},
          ${python:Depends},

diff --git a/debian/rules b/debian/rules
@@ -5,7 +5,7 @@ export DESTDIR=$(CURDIR)/debian/tmp
 
 export DEB_HOST_ARCH      ?= $(shell dpkg-architecture -qDEB_HOST_ARCH)
 
-extraopts += -DUSE_CRYPTOPP=OFF -DWITH_OCF=ON -DWITH_LTTNG=ON -DWITH_PYTHON3=ON
+extraopts += -DUSE_CRYPTOPP=OFF -DWITH_OCF=ON -DWITH_LTTNG=ON -DWITH_PYTHON3=ON -DWITH_MGR_DASHBOARD_V2_FRONTEND=OFF
 extraopts += -DWITH_CEPHFS_JAVA=ON
 # assumes that ceph is exmpt from multiarch support, so we override the libdir.
 extraopts += -DCMAKE_INSTALL_LIBDIR=/usr/lib

diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst
@@ -2,7 +2,29 @@ dashboard plugin
 ================
 
 Dashboard plugin visualizes the statistics of the cluster using a web server
-hosted by ``ceph-mgr``.
+hosted by ``ceph-mgr``. The dashboard currently provides insight into the
+following aspects of your Ceph cluster:
+
+* **Overall cluster health**: The overall cluster status, storage utilization
+  (e.g. number of objects, raw capacity, usage per pool), a list of pools and
+  their status and usage statistics, access to the cluster log file.
+* **Hosts**: A list of all hosts associated to the cluster, which services
+  are running and which version of Ceph is installed.
+* **Performance counters**: Display detailed statistics for each running service.
+* **Monitors**: List of all MONs, their quorum status, open sessions.
+* **Configuration Reference**: List all available configuration options, their
+  description and default values.
+* **OSDs**: A list of all OSDs, their status and usage statistics as well as
+  detailed information like attributes (OSD map), metadata, performance counters
+  and usage histograms for read/write operations.
+* **iSCSI**: List all hosts that run the TCMU runner service, list of all images
+  and their performance characteristics (read/write ops, traffic).
+* **RBD**: List of all RBD images and their properties (size, objects, features)
+  in a given pool. 
+* **CephFS**: List all active filesystem clients and associated pools, including
+  their usage statistics.
+* **Object Gateway**: List of all active object gateways and their performance
+  counters.
 
 Enabling
 --------
@@ -39,6 +61,23 @@ If the port is not configured, the web app will bind to port ``7000``.
 If the address it not configured, the web app will bind to ``::``,
 which corresponds to all available IPv4 and IPv6 addresses.
 
+In order to be able to log in, you need to define a username and password, which
+will be stored in the MON's configuration database::
+
+  ceph dashboard set-login-credentials <username> <password>
+
+The password will be stored in the configuration database in encrypted form
+using ``bcrypt``. This is a global setting that applies to all dashboard instances.
+
+You can now access the dashboard using your (JavaScript-enabled) web browser, by
+pointing it to the selected TCP port and any of the host names or IP addresses
+where a manager instance runs on, e.g. ``http://<$IP>:<$PORT>/``.
+
+You should then be greeted by the dashboard login page, requesting your
+previously defined username and password. Select the **Keep me logged in**
+checkbox if you want to skip the username/password request when accessing the
+dashboard in the future.
+
 Reverse proxies
 ---------------
 
@@ -51,5 +90,4 @@ to use hyperlinks that include your prefix, you can set the
 
   ceph config-key set mgr/dashboard/url_prefix $PREFIX
 
-so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.
-
+so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.
diff --git a/install-deps.sh b/install-deps.sh
@@ -19,6 +19,8 @@ if test $(id -u) != 0 ; then
 fi
 export LC_ALL=C # the following is vulnerable to i18n
 
+ARCH=`uname -m`
+
 function munge_ceph_spec_in {
     local OUTFILE=$1
     sed -e 's/@//g' -e 's/%bcond_with make_check/%bcond_without make_check/g' < ceph.spec.in > $OUTFILE
@@ -62,6 +64,68 @@ EOF
     $SUDO ln -nsf /usr/bin/g++ /usr/bin/x86_64-linux-gnu-g++
 }
 
+function ensure_min_npm_version {
+  local install_npm_pkg_cmd=$1
+
+  if [ "$ARCH" = "aarch64" ]; then
+    # we don't support dashboard frontend development in arm64 architecture
+    return 0
+  fi
+
+  if ! type npm > /dev/null 2>&1; then
+    $SUDO $install_npm_pkg_cmd
+  fi
+
+  NODE_VER=`node -v`
+  NODE_VER_MAJOR=`node -v | sed 's/v\(\w\+\).*/\1/g'`
+  NODE_VER_MINOR=`node -v | sed 's/v\w\+\.\(\w\+\).*/\1/g'`
+
+  # The minimum node version required is 4.8.0 so that we can use yarn below
+  UPDATE_NODE=false
+  if [ $NODE_VER_MAJOR -lt 4 ]; then
+    UPDATE_NODE=true
+  elif [ $NODE_VER_MAJOR -eq 4 ] && [ $NODE_VER_MINOR -lt 8 ]; then
+    UPDATE_NODE=true
+  fi
+  if $UPDATE_NODE; then
+    $SUDO npm install -g n
+    # installs nodejs version 4.8.0
+    $SUDO n 4.8.0
+    $SUDO npm uninstall -g n
+    hash -d node > /dev/null 2>&1 || true
+  fi
+
+  NPM_VER=`npm -v`
+  NPM_VER_MAJOR=`npm -v | sed 's/\(\w\+\).*/\1/g'`
+
+  # The minimum npm version required is 5.0.0 so that we can install and use
+  # a local nodejs installation (required by the dashboard angular2 frontend)
+  if [ $NPM_VER_MAJOR -lt 5 ]; then
+    $SUDO npm install -g yarn
+    $SUDO yarn global add npm@^5.0.0  # install npm version 5.0.0 or later
+  fi
+  hash -d npm > /dev/null 2>&1 || true
+
+  NEW_NODE_VER=`node -v`
+  NEW_NPM_VER=`npm -v`
+  if [ ! "$NODE_VER" = "$NEW_NODE_VER" ]; then
+cat <<EOF
+*****************************************************************************
+      YOUR NODE VERSION WAS UPDATED FROM $NODE_VER TO $NEW_NODE_VER
+*****************************************************************************
+EOF
+  fi
+  if [ ! "$NPM_VER" = "$NEW_NPM_VER" ]; then
+cat <<EOF
+*****************************************************************************
+      YOUR NPM VERSION WAS UPDATED FROM $NPM_VER TO $NEW_NPM_VER
+      TO RETURN TO VERSION $NPM_VER run the following command:
+          $ $SUDO yarn global remove npm && hash -d npm
+*****************************************************************************
+EOF
+  fi
+}
+
 if [ x`uname`x = xFreeBSDx ]; then
     $SUDO pkg install -yq \
         devel/babeltrace \
@@ -148,6 +212,11 @@ else
 	$SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
 	$SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
 	if [ -n "$backports" ] ; then rm $control; fi
+        if [ ! "$ARCH" = "aarch64" ]; then
+          $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y install nodejs
+          [ ! -e /usr/bin/node ] && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y install nodejs-legacy
+          ensure_min_npm_version "env DEBIAN_FRONTEND=noninteractive apt-get -y install npm"
+        fi
         ;;
     centos|fedora|rhel|ol|virtuozzo)
         yumdnf="yum"
@@ -186,12 +255,14 @@ else
         munge_ceph_spec_in $DIR/ceph.spec
         $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out
         ! grep -q -i error: $DIR/yum-builddep.out || exit 1
+        ensure_min_npm_version "$yumdnf install -y npm"
         ;;
     opensuse*|suse|sles)
         echo "Using zypper to install dependencies"
         $SUDO zypper --gpg-auto-import-keys --non-interactive install lsb-release systemd-rpm-macros
         munge_ceph_spec_in $DIR/ceph.spec
         $SUDO zypper --non-interactive install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
+        ensure_min_npm_version "zypper --non-interactive install npm"
         ;;
     alpine)
         # for now we need the testing repo for leveldb

diff --git a/make-dist b/make-dist
@@ -61,6 +61,15 @@ download_boost() {
     rm -rf src/boost
 }
 
+build_dashboard_frontend() {
+  CURR_DIR=`pwd`
+  cd src/pybind/mgr/dashboard_v2/frontend
+  npm install
+  npm run build -- --prod
+  cd $CURR_DIR
+  tar cf dashboard_frontend.tar $outfile/src/pybind/mgr/dashboard_v2/frontend/dist
+}
+
 # clean out old cruft...
 echo "cleanup..."
 rm -f $outfile*
@@ -105,9 +114,11 @@ download_boost $boost_version b2dfbd6c717be4a7bb2d88018eaccf75 \
                https://dl.bintray.com/boostorg/release/$boost_version/source \
                https://downloads.sourceforge.net/project/boost/boost/$boost_version \
                https://download.ceph.com/qa
+build_dashboard_frontend
 tar --concatenate -f $outfile.all.tar $outfile.version.tar
 tar --concatenate -f $outfile.all.tar $outfile.boost.tar
 tar --concatenate -f $outfile.all.tar $outfile.tar
+tar --concatenate -f $outfile.all.tar dashboard_frontend.tar
 mv $outfile.all.tar $outfile.tar
 rm $outfile
 rm -f $outfile.version.tar

diff --git a/qa/suites/rados/mgr/tasks/dashboard_v2.yaml b/qa/suites/rados/mgr/tasks/dashboard_v2.yaml
@@ -0,0 +1,33 @@
+
+tasks:
+  - install:
+  - ceph:
+      # tests may leave mgrs broken, so don't try and call into them
+      # to invoke e.g. pg dump during teardown.
+      wait-for-scrub: false
+      log-whitelist:
+        - overall HEALTH_
+        - \(MGR_DOWN\)
+        - \(PG_
+        - replacing it with standby
+        - No standby daemons available
+        - \(FS_DEGRADED\)
+        - \(MDS_FAILED\)
+        - \(MDS_DEGRADED\)
+        - \(FS_WITH_FAILED_MDS\)
+        - \(MDS_DAMAGE\)
+  - rgw: [client.0]
+  - cephfs_test_runner:
+      modules:
+        - tasks.mgr.test_dashboard_v2
+        - tasks.mgr.dashboard_v2.test_auth
+        - tasks.mgr.dashboard_v2.test_cephfs
+        - tasks.mgr.dashboard_v2.test_cluster_configuration
+        - tasks.mgr.dashboard_v2.test_dashboard
+        - tasks.mgr.dashboard_v2.test_host
+        - tasks.mgr.dashboard_v2.test_monitor
+        - tasks.mgr.dashboard_v2.test_osd
+        - tasks.mgr.dashboard_v2.test_perf_counters
+        - tasks.mgr.dashboard_v2.test_summary
+        - tasks.mgr.dashboard_v2.test_rgw
+        - tasks.mgr.dashboard_v2.test_rbd
diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py
@@ -132,7 +132,8 @@ def wait_until_equal(self, get_fn, expect_val, timeout, reject_fn=None):
 
         log.debug("wait_until_equal: success")
 
-    def wait_until_true(self, condition, timeout):
+    @classmethod
+    def wait_until_true(cls, condition, timeout):
         period = 5
         elapsed = 0
         while True:

diff --git a/qa/tasks/mgr/dashboard_v2/__init__.py b/qa/tasks/mgr/dashboard_v2/__init__.py