[OGUI-1698] Implement configuration endpoint

Control/index.js

@@ -19,6 +19,13 @@ const config = require('./lib/config/configProvider.js');
 const {buildPublicConfig} = require('./lib/config/publicConfigProvider.js');
 const api = require('./lib/api.js');
 
+
+// Initialize nock for Consul only if we are in test environment
+if (process.env.NODE_ENV === "test") {
+  const { initializeNockForConsul } = require("./test/config/testConfigForConsul.js");
+  initializeNockForConsul();
+}
+
 // -------------------------------------------------------
 
 buildPublicConfig(config);

Control/lib/api.js

@@ -25,6 +25,8 @@ const {addDetectorIdMiddleware} = require('./middleware/addDetectorId.middleware
 const {logDeploymentRequestMiddleware} = require('./middleware/logDeploymentRequest.middleware.js');
 const {minimumRoleMiddleware} = require('./middleware/minimumRole.middleware.js');
 const {requireDetectorOrGlobalRoleMiddleware} = require('./middleware/requireDetectorOrGlobalRole.middleware.js');
+const {validateConsulServiceMiddlewareFactory} = require('./middleware/validateConsulServiceMiddlewareFactory.js');
+
 const {
   setDetectorsFromEnvironmentMiddlewareFactory
 } = require('./middleware/setDetectorsFromEnvironmentMiddlewareFactory.js');
@@ -33,6 +35,7 @@ const {
 } = require('./middleware/getDetectorsLockOwnershipMiddlewareFactory.js');
 
 // controllers
+const {QCConfigurationController} = require('./controllers/QCConfiguration.controller.js');
 const {ConsulController} = require('./controllers/Consul.controller.js');
 const {DeploymentController} = require('./controllers/Deployment.controller.js');
 const {EnvironmentController} = require('./controllers/Environment.controller.js');
@@ -57,6 +60,7 @@ const {RunService} = require('./services/Run.service.js');
 const {StatusService} = require('./services/Status.service.js');
 const {TaskService} = require('./services/Task.service.js');
 const {WorkflowTemplateService} = require('./services/WorkflowTemplate.service.js');
+const {QCConfigurationService} = require('./services/QCConfiguration.service.js');
 
 // web-ui services
 const {NotificationService, ConsulService} = require('@aliceo2/web-ui');
@@ -99,8 +103,10 @@ module.exports.setup = (http, ws) => {
   const cacheService = new CacheService(broadcastService);
   const environmentCacheService = new EnvironmentCacheService(broadcastService, eventEmitter);
 
+  const qcConfigurationService = new QCConfigurationService(consulService);
+  const qcConfigurationController = new QCConfigurationController(qcConfigurationService, config.consul);
+
   const consulController = new ConsulController(consulService, config.consul);
-  consulController.testConsulStatus();
 
   const ctrlProxy = new GrpcServiceClient(config.grpc, O2_CONTROL_PROTO_PATH);
   const ctrlService = new ControlService(ctrlProxy, consulController, config.grpc, O2_CONTROL_PROTO_PATH);
@@ -161,14 +167,15 @@ module.exports.setup = (http, ws) => {
 
   const intervals = new Intervals();
 
-  initializeData(apricotService, lockService);
+  initializeData(apricotService, lockService, consulService);
   initializeIntervals(intervals, statusService, runService, bkpService, environmentService);
 
   const coreMiddleware = [
     ctrlService.isConnectionReady.bind(ctrlService),
   ];
   const setDetectorsFromEnvironmentMiddleware = setDetectorsFromEnvironmentMiddlewareFactory(environmentService);
   const verifyLockOwnershipMiddleware = getDetectorsLockOwnershipMiddlewareFactory(lockService);
+  const validateConsulServiceMiddleware = validateConsulServiceMiddlewareFactory(consulService);
 
   ctrlProxy.methods.forEach(
     (method) => http.post(`/${method}`, coreMiddleware, (req, res) => ctrlService.executeCommand(req, res)),
@@ -270,13 +277,31 @@ module.exports.setup = (http, ws) => {
     statusController.getAliECSIntegratedServicesStatus.bind(statusController),
   );
 
+  // Configuration
+  http.get(
+    '/configurations', validateConsulServiceMiddleware,
+    qcConfigurationController.getConfigurationsKeysHandler.bind(qcConfigurationController)
+  );
+  http.get(
+    '/configurations/:key(*)', validateConsulServiceMiddleware, 
+    qcConfigurationController.getConfigurationByKeyHandler.bind(qcConfigurationController)
+  );
+
   // Consul
-  const validateService = consulController.validateService.bind(consulController);
-  http.get('/consul/flps', validateService, consulController.getFLPs.bind(consulController));
-  http.get('/consul/crus', validateService, consulController.getCRUs.bind(consulController));
-  http.get('/consul/crus/config', validateService, consulController.getCRUsWithConfiguration.bind(consulController));
-  http.get('/consul/crus/aliases', validateService, consulController.getCRUsAlias.bind(consulController));
-  http.post('/consul/crus/config/save', validateService, consulController.saveCRUsConfiguration.bind(consulController));
+  http.get('/consul/flps', validateConsulServiceMiddleware, consulController.getFLPs.bind(consulController));
+  http.get('/consul/crus', validateConsulServiceMiddleware, consulController.getCRUs.bind(consulController));
+  http.get(
+    '/consul/crus/config', validateConsulServiceMiddleware, 
+    consulController.getCRUsWithConfiguration.bind(consulController)
+  );
+  http.get(
+    '/consul/crus/aliases', validateConsulServiceMiddleware, 
+    consulController.getCRUsAlias.bind(consulController)
+  );
+  http.post(
+    '/consul/crus/config/save', validateConsulServiceMiddleware, 
+    consulController.saveCRUsConfiguration.bind(consulController)
+  );
 };
 
 /**
@@ -320,8 +345,21 @@ function initializeIntervals(intervalsService, statusService, runService, bkpSer
  * Function to initialize in order dependent services
  * @param {ApricotService} apricotService - request initial set of data from AliECS/Apricot
  * @param {LockService} lockService - initialize service with data from Apricot
+ * @param {ConsulService} consulService - service for communicating with Consul
  */
-async function initializeData(apricotService, lockService) {
+async function initializeData(apricotService, lockService, consulService) {
+  testConsulStatus(consulService);
   await apricotService.init();
   lockService.setLockStatesForDetectors(apricotService.detectors);
 }
+
+/**
+ * Method to check if consul service can be used
+ * @param {ConsulService} consulService
+ */
+function testConsulStatus(consulService) {
+  consulService
+    .getConsulLeaderStatus()
+    .then((data) => logger.info(`Service is up and running on: ${data}`))
+    .catch((error) => logger.error(`Connection failed due to ${error}`));
+}

Control/lib/controllers/Consul.controller.js

@@ -39,31 +39,6 @@ class ConsulController {
     this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'cog'}/consul`);
   }
 
-  /**
-   * Check if consulService is present:
-   * * If yes, allow request to continue
-   * * If not, send response accordingly
-   * @param {Request} req
-   * @param {Response} res
-   * @param {Next} next
-   */
-  validateService(req, res, next) {
-    if (this.consulService) {
-      next();
-    } else {
-      errorHandler('Unable to retrieve configuration of consul service', res, 502);
-    }
-  }
-
-  /**
-   * Method to check if consul service can be used
-   */
-  async testConsulStatus() {
-    this.consulService.getConsulLeaderStatus()
-      .then((data) => this._logger.info(`Service is up and running on: ${data}`))
-      .catch((error) => this._logger.error(`Connection failed due to ${error}`));
-  }
-
   /**
   * Method to request all CRUs available in consul KV store under the
   * hardware key

Control/lib/controllers/QCConfiguration.controller.js

@@ -0,0 +1,98 @@
+/**
+ * @license
+ * Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+ * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+ * All rights not expressly granted are reserved.
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const {
+  LogManager,
+  updateAndSendExpressResponseFromNativeError,
+  InvalidInputError,
+  NotFoundError,
+  ServiceUnavailableError,
+} = require("@aliceo2/web-ui");
+const { errorLogger } = require("../utils.js");
+const { getConsulConfig } = require("../config/publicConfigProvider.js");
+
+/**
+ * Gateway for all Consul Consumer calls
+ */
+class QCConfigurationController {
+  /**
+   * Setup QCConfigurationController
+   * @param {QCConfigurationService} qcConfigurationService - service for managing QC configurations
+   * @param {JSON} config - consul configuration
+   */
+  constructor(qcConfigurationService, config) {
+    this._qcConfigurationService = qcConfigurationService;
+    this._config = getConsulConfig({ consul: config });
+    this._qcConfigurationsPath = `${this._config.qcPath}/ANY/any`;
+
+    this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? "cnf"}/qc-configuration-controller`);
+  }
+
+  /**
+   * Method to get configurations names
+   * @param {Request} req - HTTP Request object
+   * @param {Response} res - HTTP Response object
+   */
+  async getConfigurationsKeysHandler(req, res) {
+    const { prefix = "", recurse = false } = req.query;
+    const prefixPath = prefix ? `${this._qcConfigurationsPath}/${prefix}` : this._qcConfigurationsPath;
+
+    try {
+      const validKeys = await this._qcConfigurationService.retrieveKeysOfValidConfigurations(prefixPath, recurse);
+
+      if (!validKeys || validKeys.length === 0) {
+        updateAndSendExpressResponseFromNativeError(res, new NotFoundError("No valid configurations found"));
+        return;
+      }
+
+      res.status(200).json(validKeys);
+    } catch (error) {
+      errorLogger(error, this._logger);
+      if (error.message?.includes('Non-2xx status code: 404')) {
+        updateAndSendExpressResponseFromNativeError(res, 
+          new NotFoundError(`Configurations prefix not found: "${prefixPath}"`));
+      } else {
+        updateAndSendExpressResponseFromNativeError(res, new ServiceUnavailableError("Consul service unavailable"));
+      }
+    }
+  }
+
+  /**
+   * Method to get configuration value by key
+   * @param {Request} req - HTTP Request object
+   * @param {Response} res - HTTP Response object
+   */
+  async getConfigurationByKeyHandler(req, res) {
+    const { key } = req.params;
+
+    if (!key || key.trim() === "") {
+      updateAndSendExpressResponseFromNativeError(res, new InvalidInputError("Missing configuration key"));
+      return;
+    }
+
+    try {
+      const value = await this._qcConfigurationService.retrieveConfigurationByKey(key);
+      res.status(200).json(value);
+    } catch (error) {
+      errorLogger(error, this._logger);
+      if (error.message?.includes('Non-2xx status code: 404')) {
+        updateAndSendExpressResponseFromNativeError(res, new NotFoundError(`Configuration not found for key: ${key}`));
+      } else {
+        updateAndSendExpressResponseFromNativeError(res, new ServiceUnavailableError("Consul service unavailable"));
+      }
+    }
+  }
+}
+
+exports.QCConfigurationController = QCConfigurationController;

Control/lib/middleware/validateConsulServiceMiddlewareFactory.js

@@ -0,0 +1,39 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+
+const { updateAndSendExpressResponseFromNativeError, ServiceUnavailableError } = require("@aliceo2/web-ui");
+
+/**
+ * Factory function to check if consul service is available
+ *
+ * @param {ConsulService} consulService - service for which availability is checked
+ * @returns {function(req, res, next): void} - middleware function
+ */
+const validateConsulServiceMiddlewareFactory = (consulService) => {
+  /**
+   * Middleware function to check if consul service is available
+   * @param {Request} req - HTTP Request object
+   * @param {Response} res - HTTP Response object
+   * @param {Next} next - HTTP Next object to use if checks pass
+   * @returns {void} continue if checks pass, uses response object to respond with error if checks fail
+   */
+  return async (req, res, next) => {
+    if (consulService) {
+      next();
+    } else {
+      updateAndSendExpressResponseFromNativeError(res, new ServiceUnavailableError("Consul service is not available"));
+    }
+  };
+};
+
+exports.validateConsulServiceMiddlewareFactory = validateConsulServiceMiddlewareFactory;

Control/lib/services/QCConfiguration.service.js

@@ -0,0 +1,83 @@
+/**
+ * @license
+ * Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+ * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+ * All rights not expressly granted are reserved.
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const { LogManager } = require("@aliceo2/web-ui");
+
+/**
+ * @class
+ * QCConfigurationService class to be user for communicating with the Consul service
+ */
+class QCConfigurationService {
+  /**
+   * @constructor
+   * Constructor for configuring the initial state of stored information
+   * @param {ConsulService} consulService - service to communicate with Consul
+   */
+  constructor(consulService) {
+    /**
+     * @type {ConsulService}
+     */
+    this._consulService = consulService;
+
+    this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? "cnf"}/qc-configuration-service`);
+  }
+
+  /**
+   * Get keys of configurations stored in Consul
+   * @param {String} prefix - prefix to filter the keys
+   * @param {boolean} [recurse=false] - whether to recurse into subdirectories
+   */
+  async retrieveKeysOfValidConfigurations(prefix, recurse = false) {
+    const data = await this._consulService.getOnlyRawValuesByKeyPrefix(prefix);
+    return this.filterConfigurations(data, recurse, prefix);
+  }
+
+  /**
+   * Get configuration by key from Consul
+   * @param {string} key - the key of the configuration
+   */
+  async retrieveConfigurationByKey(key) {
+    return await this._consulService.getOnlyRawValueByKey(key);
+  }
+
+
+  /**
+   * Filters a configuration object and returns keys of entries with valid JSON values.
+   * @param {object} configs - an object with string values to be checked.
+   * @param {boolean} recurse - whether to recurse into subdirectories
+   * @param {string} prefix - the prefix to filter keys
+   */
+  filterConfigurations(configs, recurse, prefix) {
+    const parsedData = [];
+    Object.entries(configs || {}).forEach(([key, value]) => {
+      try {
+        if (!recurse && key.replace(`${prefix}/`, "").includes("/")) {
+          return;
+        }
+
+        const parsedValue = JSON.parse(value);
+
+        if (typeof parsedValue === 'object' && parsedValue !== null && !Array.isArray(parsedValue)) {
+          parsedData.push(key);
+        }
+      } catch (e) {
+        // skip
+      }
+    });
+
+    return parsedData;
+  }
+}
+
+exports.QCConfigurationService = QCConfigurationService;