Collinhva/o2 b 1109/environment filter status history

CHANGELOG.md

@@ -2,6 +2,30 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [0.76.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.76.0)
+* Notable changes for users:
+  * Overview of runs now show quality by using a line under them in green/red
+  * The link to details page from run overview page do not include all the run info anymore
+  * Environment id is now written in white in the environment overview
+  * Fixed links of attachment in log tree
+  * Added a button in runs overview to filter on PHYSICS
+  * Added the possibility to filter logs to remove anonymous
+  * Fixed LHC info that disappeared after saving the run in RunDetail
+  * Added counts of runs associated with given LHC Periods on LHC Periods page
+  * The statistics page now contains a histogram which displays the most frequent environment history combinations and their occurrences within a given timeframe
+  * Changed existing environment status acronyms in accordance to the RMreportW29_2023
+* Notable change for developers:
+  * Added possibility in run update proto request to specify user that started and user that stopped the run
+  * The GetAllLogsUseCase now supports simultaneous inclusion and exclusion filter parameters for the author filter
+  * Added Data Pass Service (MonALISA)
+  * Added a service to fetch data passes from MonALISA
+  * Added a few new env variables in config.ServicesConfig
+  * Added a service to schedule processes and applied it to the existing handling of lost envs and runs
+  * Created a reusable overview page model and applied it on LHC fills page
+  * Fixed race condition on c++ API on RHEL 8
+  * Added filters on environment API to filter on ID
+  * Use scheduled process manager for synchronisation with MonALISA
+
 ## [0.75.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.75.0)
 * Notable changes for users:
   * Added run per Period view

database/CHANGELOG.md

@@ -1,3 +1,8 @@
+## [0.76.0]
+* Changes made to the database
+  * Added unique constraint to column name of data_passes table
+  * Run entity now has optional user_o2_start and user_o2_stop, both of type Integer.
+
 ## [0.53.0]
 * Changes made to the database:
   * `tags.description` is a new property of tags which is limited to 100 characters

lib/application.js

@@ -14,11 +14,14 @@ const { Logger } = require('./utilities');
 const database = require('./database');
 const { webUiServer } = require('./server');
 const { gRPCServer } = require('./server/index.js');
-const { GRPCConfig } = require('./config');
+const { GRPCConfig, ServicesConfig } = require('./config');
+
+const { monalisa: monalisaConfig } = ServicesConfig;
 const { handleLostRunsAndEnvironments } = require('./server/services/housekeeping/handleLostRunsAndEnvironments.js');
-const { ServicesConfig } = require('./config/index.js');
 const { isInTestMode } = require('./utilities/env-utils.js');
 const { ScheduledProcessesManager } = require('./server/services/ScheduledProcessesManager.js');
+const { MonALISASynchronizer } = require('./server/monalisa-synchronization/MonALISASynchronizer');
+const { createMonALISAClient } = require('./server/monalisa-synchronization/MonALISAClient');
 
 /**
  * Bookkeeping Application
@@ -52,26 +55,20 @@ class BookkeepingApplication {
                 await this.gRPCServer.listen(gRPCOrigin);
             }
 
-            if (ServicesConfig.enableHousekeeping) {
+            if (monalisaConfig.enableSynchronization) {
+                const monALISASynchronizer = await this.createMonALISASynchronizer();
                 this.scheduledProcessesManager.schedule(
-                    async () => {
-                        try {
-                            const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments();
-                            const subMessages = [];
-                            if (transitionedEnvironments.length > 0) {
-                                subMessages.push(`environments (${transitionedEnvironments.join(', ')})`);
-                            }
-                            if (endedRuns.length > 0) {
-                                subMessages.push(`runs (${endedRuns.join(', ')})`);
-                            }
-
-                            if (subMessages.length > 0) {
-                                this.logger.debug(`Updated lost ${subMessages.join(' and ')}`);
-                            }
-                        } catch (error) {
-                            this.logger.error(`Error while handling lost runs & environments: ${error}`);
-                        }
+                    () => monALISASynchronizer.synchronizeDataPassesFromMonALISA(),
+                    {
+                        wait: 10 * 1000,
+                        every: monalisaConfig.synchronizationPeriod,
                     },
+                );
+            }
+
+            if (ServicesConfig.enableHousekeeping) {
+                this.scheduledProcessesManager.schedule(
+                    () => this.housekeeping(),
                     {
                         wait: 30 * 1000,
                         every: 30 * 1000,
@@ -86,6 +83,43 @@ class BookkeepingApplication {
         this.logger.info('Started');
     }
 
+    /**
+     * Instantiate MonALISA synchronizer with global configuration
+     * @return {MonALISASynchronizer} instance
+     */
+    async createMonALISASynchronizer() {
+        return new MonALISASynchronizer(await createMonALISAClient({
+            dataPassesUrl: monalisaConfig.dataPassesUrl,
+            dataPassDetailsUrl: monalisaConfig.dataPassDetailsUrl,
+            yearLowerLimit: monalisaConfig.dataPassesYearLowerLimit,
+            userCertificatePath: monalisaConfig.userCertificate.path,
+            certificatePassphrase: monalisaConfig.userCertificate.passphrase,
+        }));
+    }
+
+    /**
+     * Houskeeping method, it wraps @see handleLostRunsAndEnvironments and logs its results
+     * @return {Promise<void>} promise
+     */
+    async housekeeping() {
+        try {
+            const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments();
+            const subMessages = [];
+            if (transitionedEnvironments.length > 0) {
+                subMessages.push(`environments (${transitionedEnvironments.join(', ')})`);
+            }
+            if (endedRuns.length > 0) {
+                subMessages.push(`runs (${endedRuns.join(', ')})`);
+            }
+
+            if (subMessages.length > 0) {
+                this.logger.debug(`Updated lost ${subMessages.join(' and ')}`);
+            }
+        } catch (error) {
+            this.logger.error(`Error while handling lost runs & environments: ${error}`);
+        }
+    }
+
     /**
      * Begins the process of terminating the application. Calling this method terminates the process.
      *

lib/config/services.js

@@ -17,6 +17,8 @@ const {
     DATA_PASSES_YEAR_LOWER_LIMIT,
     MONALISA_DATA_PASSES_URL,
     MONALISA_DATA_PASS_DETAILS_URL,
+    MONALISA_ENABLE_SYNCHRONIZATION,
+    MONALISA_SYNCHRONIZATION_PERIODS,
 } = process.env ?? {};
 
 exports.services = {
@@ -46,5 +48,8 @@ exports.services = {
 
         dataPassesUrl: MONALISA_DATA_PASSES_URL,
         dataPassDetailsUrl: MONALISA_DATA_PASS_DETAILS_URL,
+
+        enableSynchronization: MONALISA_ENABLE_SYNCHRONIZATION?.toLowerCase() === 'true',
+        synchronizationPeriod: Number(MONALISA_SYNCHRONIZATION_PERIODS) || 3600000, //  1h in milisecond
     },
 };

lib/database/adapters/LhcPeriodStatisticsAdapter.js

@@ -33,11 +33,13 @@ class LhcPeriodStatisticsAdapter {
         const { id, avgCenterOfMassEnergy, lhcPeriod } = databaseObject;
         const distinctEnergies = databaseObject.get('distinctEnergies');
         const beamType = databaseObject.get('beamType');
+        const runsCount = databaseObject.get('runsCount');
         const entity = {
             id,
             avgCenterOfMassEnergy,
             distinctEnergies: distinctEnergies?.split(',').map((energy) => Number(energy)) ?? [],
             beamType,
+            runsCount,
         };
         entity.lhcPeriod = lhcPeriod ? this.lhcPeriodAdapter.toEntity(lhcPeriod) : null;
         return entity;

lib/database/models/typedefs/SequelizeLhcPeriodStatistics.js

@@ -18,4 +18,5 @@
  * @property {number|null} avgCenterOfMassEnergy
  * @property {number[]|null} distinctEnergies
  * @property {SequelizeLhcPeriod} lhcPeriod
+ * @property {number} runsCount
  */

lib/domain/entities/LhcPeriodStatistics.js

@@ -18,4 +18,5 @@
  * @property {number|null} avgCenterOfMassEnergy
  * @property {number[]} distinctEnergies
  * @property {LhcPeriod} lhcPeriod
+ * @property {number} runsCount
  */

lib/public/app.css

@@ -24,6 +24,10 @@ html, body {
 .b-underline-good { border-bottom: 4px solid var(--color-success); }
 .b-underline-bad { border-bottom: 4px dashed var(--color-danger); }
 
+.bg-light-blue {
+    background-color: var(--color-light-blue) !important;
+}
+
 .overflow {
     height: 1.5rem;
     overflow: hidden;

lib/public/components/common/chart/lineChart/lineChartComponent.js

@@ -38,11 +38,10 @@ const DEFAULT_TOOLTIP_MARGIN = 20;
 class LineChartClassComponent {
     /**
      * Constructor
-     * @param {{attrs: {points: Point[], configuration: LineChartComponentConfiguration, notify: function}}} the component's vnode
+     * @param {{attrs: {points: Point[], configuration: LineChartComponentConfiguration}}} the component's vnode
      */
-    constructor({ attrs: { points, configuration, notify } }) {
+    constructor({ attrs: { points, configuration } }) {
         this.configuration = configuration;
-        this.notify = notify;
         this.tooltipElement = null;
         this.hoveredPointCoordinates = null;
 
@@ -63,9 +62,8 @@ class LineChartClassComponent {
     }
 
     // eslint-disable-next-line require-jsdoc
-    onupdate({ attrs: { points, configuration, notify } }) {
+    onupdate({ attrs: { points, configuration } }) {
         this.configuration = configuration;
-        this.notify = notify;
 
         this._chartRenderer = new LineChartRenderer(configuration.chartConfiguration, points);
     }

lib/public/domain/enums/statusAcronyms.js

@@ -13,12 +13,11 @@
 
 const statusAcronyms = {
     STANDBY: 'S',
-    DEPLOYED: 'DL',
+    DEPLOYED: 'D',
     CONFIGURED: 'C',
     RUNNING: 'R',
     ERROR: 'E',
-    MIX: 'M',
-    DESTROYED: 'DS',
+    DESTROYED: 'X',
 };
 
 export const STATUS_ACRONYMS = statusAcronyms;

lib/public/views/Environments/ActiveColumns/environmentsActiveColumns.js

@@ -13,7 +13,7 @@
 
 import { formatTimestamp } from '../../../utilities/formatting/formatTimestamp.js';
 import { formatRunsList } from '../../Runs/format/formatRunsList.js';
-import { displayEnvironmentStatusHistory } from '../format/displayEnvironmentStatusHistory.js';
+import { displayEnvironmentStatusHistoryOverview } from '../format/displayEnvironmentStatusHistoryOverview.js';
 import { displayEnvironmentStatus } from '../format/displayEnvironmentStatus.js';
 import { infoLoggerButtonGroup } from '../../../components/common/selection/infoLoggerButtonGroup/infoLoggerButtonGroup.js';
 
@@ -59,7 +59,7 @@ export const environmentsActiveColumns = {
         sortable: false,
         size: 'w-30',
         title: true,
-        format: (_, environment) => displayEnvironmentStatusHistory(environment),
+        format: (_, environment) => displayEnvironmentStatusHistoryOverview(environment),
         balloon: true,
     },
     runs: {

lib/public/views/Environments/format/displayEnvironmentStatusHistory.js

@@ -10,28 +10,17 @@
  *  granted to it by virtue of its status as an Intergovernmental Organization
  *  or submit itself to any jurisdiction.
  */
-import { h } from '/js/src/index.js';
+
 import { STATUS_ACRONYMS } from '../../../domain/enums/statusAcronyms.js';
-import { coloredEnvironmentStatusComponent } from '../ColoredEnvironmentStatusComponent.js';
 
 /**
- * Display the given environment's status history string
+ * Display the given environment's status history as a string of acronyms
  *
- * @param {Environment} environment the environment for which status should be displayed
- * @return {vnode} the resulting component
+ * @param {String} statusHistory the environment status history seperated by commas
+ * @return {String} the abbreviated string representation of the environment status history
  */
-export const displayEnvironmentStatusHistory = (environment) => {
-    const { historyItems } = environment;
-
-    const statusHistory = historyItems
-        .filter(({ status }) => status in STATUS_ACRONYMS)
-        .map(({ status }) => ({ status, content: STATUS_ACRONYMS[status] }));
-
-    return h(
-        '.flex-row',
-        statusHistory.map((value, index) => h('.flex-row', [
-            coloredEnvironmentStatusComponent(value.status, value.content),
-            h('', index < statusHistory.length - 1 ? '-' : ''),
-        ])),
-    );
-};
+export const displayEnvironmentStatusHistory = (statusHistory) =>
+    statusHistory.split(',')
+        .filter((status) => status in STATUS_ACRONYMS)
+        .map((status) => STATUS_ACRONYMS[status])
+        .join('');

lib/public/views/Environments/format/displayEnvironmentStatusHistoryOverview.js

@@ -0,0 +1,37 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+import { h } from '/js/src/index.js';
+import { STATUS_ACRONYMS } from '../../../domain/enums/statusAcronyms.js';
+import { coloredEnvironmentStatusComponent } from '../ColoredEnvironmentStatusComponent.js';
+
+/**
+ * Display the given environment's status history string
+ *
+ * @param {Environment} environment the environment for which status should be displayed
+ * @return {vnode} the resulting component
+ */
+export const displayEnvironmentStatusHistoryOverview = (environment) => {
+    const { historyItems } = environment;
+
+    const statusHistory = historyItems
+        .filter(({ status }) => status in STATUS_ACRONYMS)
+        .map(({ status }) => ({ status, content: STATUS_ACRONYMS[status] }));
+
+    return h(
+        '.flex-row',
+        statusHistory.map((value, index) => h('.flex-row', [
+            coloredEnvironmentStatusComponent(value.status, value.content),
+            h('', index < statusHistory.length - 1 ? '-' : ''),
+        ])),
+    );
+};

lib/public/views/Runs/Details/RunDetailsModel.js

@@ -143,7 +143,8 @@ export class RunDetailsModel extends Observable {
             body: JSON.stringify(this.runPatch.toPojo()),
         };
 
-        await this._handleFetchRemoteRun(() => jsonFetch(`/api/runs/${runNumber}`, options));
+        await jsonFetch(`/api/runs/${runNumber}`, options);
+        await this._fetchOneRun();
     }
 
     /**
@@ -433,10 +434,10 @@ export class HostTreeNodeModel extends CollapsibleTreeNodeModel {
         try {
             const { data: processesExecutions } =
                 await getRemoteData('/api/dpl-process/executions?'
-                                    + `runNumber=${this._runNumber}`
-                                    + `&detectorId=${this._detectorId}`
-                                    + `&processId=${this._processId}`
-                                    + `&hostId=${hostId}`);
+                    + `runNumber=${this._runNumber}`
+                    + `&detectorId=${this._detectorId}`
+                    + `&processId=${this._processId}`
+                    + `&hostId=${hostId}`);
             this.children = RemoteData.success(processesExecutions);
         } catch (error) {
             this.children = RemoteData.failure(error);
@@ -472,9 +473,9 @@ export class DplProcessTreeNodeModel extends CollapsibleTreeNodeModel {
         try {
             const { data: hosts } =
                 await getRemoteData('/api/dpl-process/hosts?'
-                                    + `runNumber=${this._runNumber}`
-                                    + `&detectorId=${this._detectorId}`
-                                    + `&processId=${processId}`);
+                    + `runNumber=${this._runNumber}`
+                    + `&detectorId=${this._detectorId}`
+                    + `&processId=${processId}`);
             this.children = RemoteData.success(hosts.map((host) => {
                 const nodeModel = new HostTreeNodeModel(host, this._runNumber, this._detectorId, processId);
                 nodeModel.bubbleTo(this);

lib/public/views/Statistics/StatisticsPage.js

@@ -24,6 +24,7 @@ import { ChartDarkColors } from './chartColors.js';
 import { tagOccurrencesBarChartComponent } from './charts/tagOccurrencesBarChartComponent.js';
 import { timeRangeFilter } from '../../components/Filters/common/filters/timeRangeFilter.js';
 import { eorReasonOccurrencesBarChartComponent } from './charts/eorReasonOccurrencesBarChartComponent.js';
+import { environmentHistoryOccurrencesBarChartComponent } from './charts/environmentHistoryOccurrencesBarChartComponent.js';
 import { formatTimeRange } from '../../components/common/formatting/formatTimeRange.js';
 import spinner from '../../components/common/spinner.js';
 import errorAlert from '../../components/common/errorAlert.js';
@@ -57,6 +58,7 @@ export const StatisticsPage = ({ statisticsModel }) => {
                 [STATISTICS_PANELS_KEYS.EFFICIENCY_PER_DETECTOR]: 'Detector efficiency',
                 [STATISTICS_PANELS_KEYS.LOG_TAG_OCCURRENCES]: 'Tag occurrences in logs',
                 [STATISTICS_PANELS_KEYS.EOR_REASON_OCCURRENCES]: 'End of run reason occurrences',
+                [STATISTICS_PANELS_KEYS.ENVIRONMENT_HISTORY_OCCURRENCES]: 'Environment history occurrences',
             },
             {
                 [STATISTICS_PANELS_KEYS.LHC_FILL_EFFICIENCY]: (remoteData) => remoteDataDisplay(remoteData, {
@@ -160,6 +162,12 @@ export const StatisticsPage = ({ statisticsModel }) => {
                         h('.flex-grow.chart-box', eorReasonOccurrencesBarChartComponent(eorReasonOccurrences)),
                     ],
                 }),
+                [STATISTICS_PANELS_KEYS.ENVIRONMENT_HISTORY_OCCURRENCES]: (remoteData) => remoteDataDisplay(remoteData, {
+                    Success: (environmentHistoryOccurrences) => [
+                        h('h3', `Environment history occurrences - ${periodLabel}`),
+                        h('.flex-grow.chart-box', environmentHistoryOccurrencesBarChartComponent(environmentHistoryOccurrences)),
+                    ],
+                }),
             },
             { panelClass: ['p2', 'g3', 'flex-column', 'flex-grow'] },
         ),