Merge pull request #989

dird: `status subscriptions` shows more details about subscription units
bareos · Dec 16, 2021 · 238efd8 · 238efd8
2 parents 248a2ea + 863a83e
commit 238efd8
Show file tree

Hide file tree

Showing 29 changed files with 644 additions and 84 deletions.
diff --git a/.bareos-check-sources-ignore b/.bareos-check-sources-ignore
@@ -13,3 +13,5 @@ webui/public/js/partials/*.min.*
 webui/public/js/bootstrap-select.js.map
 webui/public/css/*.min.*
 webui/public/css/bootstrap*.css.map
+systemtests/tests/bareos/expected/status-subscriptions.txt
+systemtests/tests/bareos/expected/status-subscriptions-detail.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -154,6 +154,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 - webui: introduce confirmation dialog on rerun and cancel job actions [PR #951]
 - docs: update documentation of config data types [PR #962]
 - Allow `release` command to be used in scripts [PR #961]
+- Show more details about subscriptions and units data with `status subscriptions` [PR #989]
 - webui: format total jobs and total files language-sensitive [PR #996]
 - webui: optimize bvfs update cache calls [PR #999]
 - Replace complicated memory pool allocator with plain malloc() [PR #997]
@@ -346,6 +347,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https:
 [PR #981]: https://github.com/bareos/bareos/pull/981
 [PR #982]: https://github.com/bareos/bareos/pull/982
 [PR #985]: https://github.com/bareos/bareos/pull/985
+[PR #989]: https://github.com/bareos/bareos/pull/989
 [PR #992]: https://github.com/bareos/bareos/pull/992
 [PR #995]: https://github.com/bareos/bareos/pull/995
 [PR #996]: https://github.com/bareos/bareos/pull/996

diff --git a/core/src/cats/bdb_query_enum_class.h b/core/src/cats/bdb_query_enum_class.h
@@ -87,7 +87,11 @@ class BareosDbQueryEnum {
     list_volumes_select_long_0 = 76,
     bvfs_lock_pathhierarchy_0 = 77,
     bvfs_unlock_tables_0 = 78,
-    SQL_QUERY_NUMBER = 79
+    subscription_select_backup_unit_overview_0 = 79,
+    subscription_select_backup_unit_total_1 = 80,
+    subscription_select_unclassified_client_fileset_0 = 81,
+    subscription_select_unclassified_amount_data_0 = 82,
+    SQL_QUERY_NUMBER = 83
   };
 };
 

diff --git a/core/src/cats/bdb_query_names.inc b/core/src/cats/bdb_query_names.inc
@@ -81,5 +81,9 @@ const char *BareosDb::query_names[] = {
 "list_volumes_select_long_0",
 "bvfs_lock_pathhierarchy_0",
 "bvfs_unlock_tables_0",
+"subscription_select_backup_unit_overview_0",
+"subscription_select_backup_unit_total_1",
+"subscription_select_unclassified_client_fileset_0",
+"subscription_select_unclassified_amount_data_0",
 NULL
 };
diff --git a/core/src/cats/ddl/creates/postgresql.sql b/core/src/cats/ddl/creates/postgresql.sql
@@ -485,4 +485,118 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
 --   but prevents errors if create script is called multiple times
 DELETE FROM Version WHERE VersionId<=2210;
 INSERT INTO Version (VersionId) VALUES (2210);
+
+-- subscription status part
+DROP VIEW IF EXISTS backup_unit_overview;
+DROP VIEW IF EXISTS latest_full_size_categorized;
+
+CREATE VIEW latest_full_size_categorized AS
+SELECT
+  c.name AS client,
+  f.fileset AS fileset,
+  j.jobbytes / 1000000 AS total_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN j.jobbytes / 1000000
+  END AS db_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN j.jobbytes / 1000000
+  END AS vm_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN j.jobbytes / 1000000
+  END AS filer_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN j.jobbytes / 1000000
+  END AS normal_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%Plugin%=%' OR f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    ELSE j.jobbytes / 1000000
+  END AS unknown_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%Plugin%=%' OR f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN
+	CASE WHEN LENGTH(f.filesettext) < 10 THEN '<empty>' ELSE f.filesettext END
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN '"' || f.filesettext || '"'
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    ELSE 
+	CASE WHEN LENGTH(f.filesettext) < 10 THEN '<empty>' ELSE f.filesettext END
+  END AS filesettext
+FROM job j
+INNER JOIN client c
+  ON c.clientid = j.clientid
+INNER JOIN fileset f
+  ON j.filesetid = f.filesetid
+WHERE jobid IN (
+  SELECT
+    MAX(j.jobid) AS jobid
+  FROM job j
+  INNER JOIN fileset f
+    ON j.filesetid = f.filesetid
+  WHERE level = 'F'
+    AND j.jobstatus IN ('T','W')
+    AND j.type = 'B'
+    AND j.jobbytes > 0
+  GROUP BY j.clientid, f.fileset
+)
+;
+
+CREATE VIEW backup_unit_overview AS
+SELECT
+  client,
+  '<all file-based>' AS fileset,
+  NULL AS db_units,
+  NULL AS vm_units,
+  CEIL(SUM(filer_mb) / 1000000) AS filer_units,
+  CASE
+    WHEN SUM(normal_mb) <= 10000000 THEN 1
+    ELSE CEIL(SUM(normal_mb)/1000000) - 9
+  END AS normal_units
+FROM latest_full_size_categorized
+WHERE normal_mb IS NOT NULL
+   OR filer_mb IS NOT NULL
+GROUP BY client
+UNION
+SELECT
+  client,
+  fileset,
+  CASE
+    WHEN db_mb > 0 THEN 1
+  END AS db_units,
+  CASE
+    WHEN vm_mb > 10000000 THEN CEIL(vm_mb / 1000000) - 9
+    WHEN vm_mb > 0 THEN 1
+  END AS vm_units,
+  NULL AS filer_units,
+  NULL AS normal_units
+FROM latest_full_size_categorized
+WHERE (db_mb IS NOT NULL AND db_mb > 0)
+   OR (vm_mb IS NOT NULL AND vm_mb > 0)
+;
+
+
+
 -- Make sure we have appropriate permissions
diff --git a/core/src/cats/ddl/grants/postgresql-readonly.sql b/core/src/cats/ddl/grants/postgresql-readonly.sql
@@ -29,3 +29,5 @@ GRANT SELECT ON NDMPJobEnvironment TO @DB_USER@;
 GRANT SELECT ON DeviceStats TO @DB_USER@;
 GRANT SELECT ON JobStats TO @DB_USER@;
 GRANT SELECT ON TapeAlerts TO @DB_USER@;
+GRANT SELECT ON backup_unit_overview TO @DB_USER@;
+GRANT SELECT ON latest_full_size_categorized TO @DB_USER@;
diff --git a/core/src/cats/ddl/grants/postgresql.sql b/core/src/cats/ddl/grants/postgresql.sql
@@ -47,3 +47,7 @@ GRANT SELECT, UPDATE ON location_locationid_seq TO @DB_USER@;
 GRANT SELECT, UPDATE ON locationlog_loclogid_seq TO @DB_USER@;
 GRANT SELECT, UPDATE ON log_logid_seq TO @DB_USER@;
 GRANT SELECT, UPDATE ON restoreobject_restoreobjectid_seq TO @DB_USER@;
+
+-- For views
+GRANT ALL ON backup_unit_overview TO @DB_USER@;
+GRANT ALL ON latest_full_size_categorized TO @DB_USER@;
diff --git a/core/src/cats/ddl/updates/postgresql.2192_2210.sql b/core/src/cats/ddl/updates/postgresql.2192_2210.sql
@@ -3,6 +3,118 @@
 BEGIN;
 CREATE INDEX job_media_media_id_job_id_idx ON jobmedia (mediaid, jobid);
 UPDATE Version SET VersionId = 2210;
+
+-- subscription status part
+DROP VIEW IF EXISTS backup_unit_overview;
+DROP VIEW IF EXISTS latest_full_size_categorized;
+
+CREATE VIEW latest_full_size_categorized AS
+SELECT
+  c.name AS client,
+  f.fileset AS fileset,
+  j.jobbytes / 1000000 AS total_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN j.jobbytes / 1000000
+  END AS db_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN j.jobbytes / 1000000
+  END AS vm_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN j.jobbytes / 1000000
+  END AS filer_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN j.jobbytes / 1000000
+  END AS normal_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%Plugin%=%' OR f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN j.jobbytes / 1000000
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    ELSE j.jobbytes / 1000000
+  END AS unknown_mb,
+  CASE
+    WHEN f.filesettext ILIKE '%{%{%File%=%Plugin%=%' OR f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN
+	CASE WHEN LENGTH(f.filesettext) < 10 THEN '<empty>' ELSE f.filesettext END
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%File%=%' THEN '"' || f.filesettext || '"'
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-percona%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-postgres%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ldap%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%mssqlvdi:%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-vmware%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-ovirt%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%meta%=' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%Plugin%=%python%:module_name=bareos-fd-libcloud%' THEN NULL
+    WHEN f.filesettext ILIKE '%{%{%File%=%' THEN NULL
+    ELSE 
+	CASE WHEN LENGTH(f.filesettext) < 10 THEN '<empty>' ELSE f.filesettext END
+  END AS filesettext
+FROM job j
+INNER JOIN client c
+  ON c.clientid = j.clientid
+INNER JOIN fileset f
+  ON j.filesetid = f.filesetid
+WHERE jobid IN (
+  SELECT
+    MAX(j.jobid) AS jobid
+  FROM job j
+  INNER JOIN fileset f
+    ON j.filesetid = f.filesetid
+  WHERE level = 'F'
+    AND j.jobstatus = 'T'
+    AND j.type = 'B'
+    AND j.jobbytes > 0
+  GROUP BY j.clientid, f.fileset
+)
+;
+
+CREATE VIEW backup_unit_overview AS
+SELECT
+  client,
+  '<all file-based>' AS fileset,
+  NULL AS db_units,
+  NULL AS vm_units,
+  CEIL(SUM(filer_mb) / 1000000) AS filer_units,
+  CASE
+    WHEN SUM(normal_mb) <= 10000000 THEN 1
+    ELSE CEIL(SUM(normal_mb)/1000000) - 9
+  END AS normal_units
+FROM latest_full_size_categorized
+WHERE normal_mb IS NOT NULL
+   OR filer_mb IS NOT NULL
+GROUP BY client
+UNION
+SELECT
+  client,
+  fileset,
+  CASE
+    WHEN db_mb > 0 THEN 1
+  END AS db_units,
+  CASE
+    WHEN vm_mb > 10000000 THEN CEIL(vm_mb / 1000000) - 9
+    WHEN vm_mb > 0 THEN 1
+  END AS vm_units,
+  NULL AS filer_units,
+  NULL AS normal_units
+FROM latest_full_size_categorized
+WHERE (db_mb IS NOT NULL AND db_mb > 0)
+   OR (vm_mb IS NOT NULL AND vm_mb > 0)
+;
+
 COMMIT;
 set client_min_messages = warning;
 ANALYSE;
diff --git a/core/src/cats/dml/0080_subscription_select_backup_unit_overview_0 b/core/src/cats/dml/0080_subscription_select_backup_unit_overview_0
@@ -0,0 +1,13 @@
+SELECT * FROM (
+  SELECT *
+  FROM backup_unit_overview
+  ORDER BY client, fileset
+) subq_a
+UNION ALL
+SELECT 'TOTAL' as client,
+       NULL as fileset,
+       SUM(db_units) as db_units,
+       SUM(vm_units) as vm_units,
+       SUM(filer_units) as filer_units,
+       SUM(normal_units) as normal_units
+FROM backup_unit_overview;
diff --git a/core/src/cats/dml/0081_subscription_select_backup_unit_total_1 b/core/src/cats/dml/0081_subscription_select_backup_unit_total_1
@@ -0,0 +1,10 @@
+SELECT
+  *, configured - used as remaining
+FROM (SELECT
+  COALESCE(SUM(db_units), 0)
+    + COALESCE(SUM(vm_units), 0)
+    + COALESCE(SUM(filer_units), 0)
+    + COALESCE(SUM(normal_units), 0)
+    AS used,
+  %d AS configured
+  FROM backup_unit_overview) subquery_a;
diff --git a/core/src/cats/dml/0082_subscription_select_unclassified_client_fileset_0 b/core/src/cats/dml/0082_subscription_select_unclassified_client_fileset_0
@@ -0,0 +1,4 @@
+SELECT client, fileset, filesettext 
+FROM latest_full_size_categorized 
+WHERE unknown_mb > 0 
+ORDER BY client, fileset;
diff --git a/core/src/cats/dml/0083_subscription_select_unclassified_amount_data_0 b/core/src/cats/dml/0083_subscription_select_unclassified_amount_data_0
@@ -0,0 +1,4 @@
+SELECT
+  SUM(unknown_mb) AS unknown_mb,
+  ROUND(sum(unknown_mb)/sum(total_mb) * 100, 2) AS unknown_percentage
+FROM latest_full_size_categorized;
diff --git a/core/src/cats/mysql_queries.inc b/core/src/cats/mysql_queries.inc
@@ -1035,5 +1035,48 @@ const char *BareosDbMysql::query_definitions[] = {
 "UNLOCK TABLES "
 ,
 
+/* 0080_subscription_select_backup_unit_overview_0 */
+"SELECT * FROM ( "
+  "SELECT * "
+  "FROM backup_unit_overview "
+  "ORDER BY client, fileset "
+") subq_a "
+"UNION ALL "
+"SELECT 'TOTAL' as client, "
+       "NULL as fileset, "
+       "SUM(db_units) as db_units, "
+       "SUM(vm_units) as vm_units, "
+       "SUM(filer_units) as filer_units, "
+       "SUM(normal_units) as normal_units "
+"FROM backup_unit_overview; "
+,
+
+/* 0081_subscription_select_backup_unit_total_1 */
+"SELECT "
+  "*, configured - used as remaining "
+"FROM (SELECT "
+  "COALESCE(SUM(db_units), 0) "
+    "+ COALESCE(SUM(vm_units), 0) "
+    "+ COALESCE(SUM(filer_units), 0) "
+    "+ COALESCE(SUM(normal_units), 0) "
+    "AS used, "
+  "%d AS configured "
+  "FROM backup_unit_overview) subquery_a; "
+,
+
+/* 0082_subscription_select_unclassified_client_fileset_0 */
+"SELECT client, fileset, filesettext "
+"FROM latest_full_size_categorized "
+"WHERE unknown_mb > 0 "
+"ORDER BY client, fileset; "
+,
+
+/* 0083_subscription_select_unclassified_amount_data_0 */
+"SELECT "
+  "SUM(unknown_mb) AS unknown_mb, "
+  "ROUND(sum(unknown_mb)/sum(total_mb) * 100, 2) AS unknown_percentage "
+"FROM latest_full_size_categorized; "
+,
+
 NULL
 };