From 6bcfa1ffb3a0ad0c94dcc97706c9a7b6f1549c79 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Wed, 16 Feb 2022 14:42:19 +0000 Subject: [PATCH 01/16] smells --- include/fdp/objects/config.hxx | 18 +++++++++++------- include/fdp/registry/api.hxx | 2 +- src/objects/config.cxx | 20 ++++++++++---------- src/registry/api.cxx | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index d2acb90..db54d17 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -31,6 +31,11 @@ namespace FDP { * */ class Config { + public: + typedef std::shared_ptr< Config > sptr; + + typedef std::map< std::string, IOObject > map_type; + private: const ghc::filesystem::path config_file_path_; const ghc::filesystem::path config_dir_; @@ -59,11 +64,11 @@ namespace FDP { ApiObject::sptr code_run_; - std::map writes_; - std::map reads_; + map_type writes_; + map_type reads_; - std::map outputs_; - std::map inputs_; + map_type outputs_; + map_type inputs_; RESTAPI rest_api_location_ = RESTAPI::LOCAL; @@ -87,7 +92,6 @@ namespace FDP { public: - typedef std::shared_ptr< Config > sptr; static Config::sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, @@ -225,7 +229,7 @@ namespace FDP { * @param data_product * @return ghc::filesystem::path */ - ghc::filesystem::path link_write(std::string &data_product); + ghc::filesystem::path link_write( const std::string &data_product); /** * @brief Return the filepath to a given data product @@ -233,7 +237,7 @@ namespace FDP { * @param data_product * @return ghc::filesystem::path */ - ghc::filesystem::path link_read(std::string &data_product); + ghc::filesystem::path link_read(const std::string& data_product); /** * @brief Finalise the pipeline diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index b5d8e37..a98665a 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -181,7 +181,7 @@ private: ghc::filesystem::path out_path); }; -std::string url_encode(std::string url); +std::string url_encode(const std::string& url); }; // namespace FDP diff --git a/src/objects/config.cxx b/src/objects/config.cxx index c745ded..d791c04 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -352,7 +352,7 @@ std::string Config::get_code_run_uuid() const{ }; -ghc::filesystem::path Config::link_write(std::string &data_product){ +ghc::filesystem::path Config::link_write( const std::string& data_product){ if (!config_has_writes()){ APILogger->error("Config Error: Write has not been specified in the given config file"); throw config_parsing_error("Config Error: Write has not been specified in the given config file"); @@ -429,7 +429,7 @@ ghc::filesystem::path Config::link_write(std::string &data_product){ } -ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ +ghc::filesystem::path FDP::Config::link_read( const std::string &data_product){ YAML::Node currentRead; auto it = inputs_.find("data_product"); @@ -551,9 +551,9 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ void FDP::Config::finalise(){ if(has_writes()){ - std::map::iterator it; + Config::map_type::iterator it; for (it = writes_.begin(); it != writes_.end(); it++){ - IOObject currentWrite = it->second; + IOObject& currentWrite = it->second; if(! file_exists(currentWrite.get_path().string())){ APILogger->error("File Error: Cannot Find file for write", currentWrite.get_use_data_product()); @@ -667,9 +667,9 @@ void FDP::Config::finalise(){ } if(has_reads()){ - std::map::iterator it; + map_type::iterator it; for (it = reads_.begin(); it != reads_.end(); it++){ - IOObject currentRead = it->second; + IOObject& currentRead = it->second; inputs_[currentRead.get_data_product()] = currentRead; } } @@ -677,9 +677,9 @@ void FDP::Config::finalise(){ Json::Value patch_data; if(has_outputs()){ - std::map::iterator it; + map_type::iterator it; for (it = outputs_.begin(); it != outputs_.end(); it++){ - IOObject currentOutput = it->second; + IOObject& currentOutput = it->second; Json::Value output = currentOutput.get_component_object()->get_uri(); patch_data["outputs"].append(output); APILogger->info("Writing {} to local registry", currentOutput.get_use_data_product()); @@ -687,9 +687,9 @@ void FDP::Config::finalise(){ } if(has_inputs()){ - std::map::iterator it; + map_type::iterator it; for (it = inputs_.begin(); it != inputs_.end(); it++){ - IOObject currentInput = it->second; + IOObject& currentInput = it->second; Json::Value input = currentInput.get_component_object()->get_uri(); patch_data["inputs"].append(input); APILogger->info("Writing {} to local registry", currentInput.get_use_data_product()); diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 3d4904a..858e429 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -11,7 +11,7 @@ static size_t write_file_(void *ptr, size_t size, size_t nmemb, FILE *stream) { return written_n_; } -std::string url_encode(std::string url) { +std::string url_encode( const std::string& url) { CURL *curl_ = curl_easy_init(); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); return curl_easy_escape(curl_, url.c_str(), 0); From e6a08b4229373331478697c73174545c8da9c8dc Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Wed, 16 Feb 2022 21:22:24 +0000 Subject: [PATCH 02/16] Correct curl callback prototype & const smell --- include/fdp/registry/api.hxx | 4 +++- src/registry/api.cxx | 14 ++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index a98665a..84ec597 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -39,6 +39,7 @@ enum class RESTAPI { LOCAL /*!< Run from local registry */ }; +#if 0 /*! ************************************************************************** * @brief function for writing to file from a CURL call * @author K. Zarebski (UKAEA) @@ -50,6 +51,7 @@ enum class RESTAPI { * @return size_t ****************************************************************************/ size_t write_func_(void *ptr, size_t size, size_t nmemb, std::string *data); +#endif /*! ************************************************************************** * @class API @@ -70,7 +72,7 @@ public: * * @param url_root the root of the query address, e.g. localhost:8000/api ***************************************************************************/ - API(std::string url_root) + API( const std::string& url_root) : url_root_(API::append_with_forward_slash(url_root)) {} /** diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 858e429..b353408 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -1,14 +1,16 @@ #include "fdp/registry/api.hxx" namespace FDP { -static size_t write_str_(void *ptr, size_t size, size_t nmemb, std::string *data) { - data->append((char *)ptr, size * nmemb); - return size * nmemb; +static size_t write_str_(char *ptr, size_t size, size_t nmemb, void* userdata ) { + std::string* data = static_cast< std::string* >( userdata ); + data->append((char *)ptr, size * nmemb); + return size * nmemb; } -static size_t write_file_(void *ptr, size_t size, size_t nmemb, FILE *stream) { - size_t written_n_ = fwrite(ptr, size, nmemb, stream); - return written_n_; +static size_t write_file_(char*ptr, size_t size, size_t nmemb, void* userdata ) { + FILE* stream = static_cast< FILE* >( userdata ); + size_t written_n_ = fwrite(ptr, size, nmemb, stream); + return written_n_; } std::string url_encode( const std::string& url) { From e3759617758e5124718597c5ae131b050f182976 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Wed, 16 Feb 2022 21:40:41 +0000 Subject: [PATCH 03/16] const smells --- include/fdp/objects/api_object.hxx | 4 ++-- include/fdp/objects/io_object.hxx | 4 ++-- include/fdp/utilities/logging.hxx | 1 - src/fdp.cxx | 7 ++----- src/objects/api_object.cxx | 4 ++-- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/include/fdp/objects/api_object.hxx b/include/fdp/objects/api_object.hxx index b221402..a810b16 100644 --- a/include/fdp/objects/api_object.hxx +++ b/include/fdp/objects/api_object.hxx @@ -49,7 +49,7 @@ namespace FDP { * @return int object id e.g. if the uri is http://127.0.0.1/object/1 * the object id will be 1 */ - int get_id(); + int get_id() const ; /** * @brief Get the object type from the uri * @@ -58,7 +58,7 @@ namespace FDP { */ static int get_id_from_string(std::string url); - std::string get_type(); + std::string get_type() const ; /** * @brief Get the object uri * diff --git a/include/fdp/objects/io_object.hxx b/include/fdp/objects/io_object.hxx index 19bb5c0..e18e214 100644 --- a/include/fdp/objects/io_object.hxx +++ b/include/fdp/objects/io_object.hxx @@ -123,7 +123,7 @@ namespace FDP { * * @return const std::string& */ - const std::string& get_use_namespace() {return use_namespace_;} + const std::string& get_use_namespace() const {return use_namespace_;} /** * @brief Get the path of the data product @@ -144,7 +144,7 @@ namespace FDP { * * @return std::string */ - const std::string& get_component_description() {return component_description_;} + const std::string& get_component_description() const {return component_description_;} /** * @brief Check whether the data product is public diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index 9a8e504..a495cfb 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -20,7 +20,6 @@ namespace FDP { { public: typedef std::shared_ptr sptr; - private: }; /** diff --git a/src/fdp.cxx b/src/fdp.cxx index 6dadaa1..95197b3 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -35,7 +35,7 @@ namespace FDP { ghc::filesystem::path config_file_path_() const {return config_->get_config_file_path();} ghc::filesystem::path script_file_path_() const {return config_->get_script_file_path();} std::string token_() const {return config_->get_token();} - RESTAPI api_location_() {return config_->get_rest_api_location();} + RESTAPI api_location_() const {return config_->get_rest_api_location();} impl& operator=(const impl& ) = delete; @@ -56,7 +56,7 @@ namespace FDP { spdlog::level::level_enum log_level = spdlog::level::info, RESTAPI api_location = RESTAPI::LOCAL); - //impl(const impl &dp) delete; + impl(const impl &dp) = default; /** * @brief Default Destructor @@ -65,9 +65,6 @@ namespace FDP { ~impl() = default; - //DataPipelineImpl_& operator=(DataPipelineImpl_ dp); - - /** * @brief Return a path to a given data product * Whilst recording it's meta data for the code run diff --git a/src/objects/api_object.cxx b/src/objects/api_object.cxx index a8b459a..f930312 100644 --- a/src/objects/api_object.cxx +++ b/src/objects/api_object.cxx @@ -62,7 +62,7 @@ namespace FDP { * * @return int of the object id */ - int ApiObject::get_id(){ + int ApiObject::get_id() const { return get_id_from_string(get_uri()); } @@ -78,7 +78,7 @@ namespace FDP { * * @return std::string The object type */ - std::string ApiObject::get_type(){ + std::string ApiObject::get_type() const { std::string uri_ = get_uri(); std::size_t id_pos = uri_.find_last_of("/"); std::string uri_prefix = uri_.substr(0, id_pos); From 81c70ed88dad960faf37738a9e77f9a920f5e7ff Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Thu, 17 Feb 2022 10:24:45 +0000 Subject: [PATCH 04/16] Remove unnecessary header --- src/registry/data_io.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index 7a17c51..1be20f1 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -1,5 +1,5 @@ #include "fdp/registry/data_io.hxx" -#include "fdp/objects/config.hxx" + namespace FDP { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { APILogger->debug("LocalFileSystem: Reading configuration file '{0}'", @@ -38,4 +38,4 @@ double read_point_estimate_from_toml(const ghc::filesystem::path var_address) { std::string get_first_key_(const toml::value data_table) { return data_table.as_table().begin()->first; } -}; // namespace FDP \ No newline at end of file +}; // namespace FDP From 2ef29818ea4cac1b43087de6b3cb1862a9c64e3a Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Thu, 17 Feb 2022 10:41:29 +0000 Subject: [PATCH 05/16] smell : FDP namespace -> fdp --- include/fdp/exceptions.hxx | 4 +-- include/fdp/fdp.hxx | 4 +-- include/fdp/objects/api_object.hxx | 2 +- include/fdp/objects/config.hxx | 2 +- include/fdp/objects/io_object.hxx | 2 +- include/fdp/objects/metadata.hxx | 4 +-- include/fdp/registry/api.hxx | 6 ++--- include/fdp/registry/data_io.hxx | 4 +-- include/fdp/utilities/json.hxx | 4 +-- include/fdp/utilities/logging.hxx | 4 +-- include/fdp/utilities/semver.hxx | 6 ++--- src/fdp.cxx | 22 ++++++++-------- src/objects/api_object.cxx | 2 +- src/objects/config.cxx | 42 +++++++++++++++--------------- src/objects/metadata.cxx | 4 +-- src/registry/api.cxx | 4 +-- src/registry/data_io.cxx | 4 +-- src/utilities/json.cxx | 4 +-- src/utilities/logging.cxx | 4 +-- src/utilities/semver.cxx | 4 +-- test/test_api.cxx | 4 +-- test/test_api_object.cxx | 2 +- test/test_config.cxx | 2 +- test/test_pimpl.cxx | 2 +- test/test_registry.cxx | 2 +- test/test_utilities.cxx | 2 +- 26 files changed, 73 insertions(+), 73 deletions(-) diff --git a/include/fdp/exceptions.hxx b/include/fdp/exceptions.hxx index 9c379e2..c6a6516 100644 --- a/include/fdp/exceptions.hxx +++ b/include/fdp/exceptions.hxx @@ -4,7 +4,7 @@ #include #include -namespace FDP { +namespace fdp { class config_parsing_error : public std::runtime_error { public: config_parsing_error(const std::string& message) @@ -37,6 +37,6 @@ public: write_error(const std::string& message) : std::runtime_error(message) {} }; -}; // namespace FDP +}; // namespace fdp #endif diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 7a329e3..a172efa 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -6,7 +6,7 @@ //#include "fdp/registry/datapipeline.hxx" -namespace FDP { +namespace fdp { /** * @brief DataPipeline Class: * A PIMPL Class for interacting the the FAIR Data Pipeline @@ -72,6 +72,6 @@ namespace FDP { std::shared_ptr< DataPipeline::impl > pimpl_; }; -}; // namespace FDP +}; // namespace fdp #endif diff --git a/include/fdp/objects/api_object.hxx b/include/fdp/objects/api_object.hxx index a810b16..05fa55b 100644 --- a/include/fdp/objects/api_object.hxx +++ b/include/fdp/objects/api_object.hxx @@ -7,7 +7,7 @@ #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace fdp { /** * @brief Class for API objects * diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index db54d17..ded5fa7 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -25,7 +25,7 @@ #include "fdp/objects/api_object.hxx" #include "fdp/objects/io_object.hxx" -namespace FDP { +namespace fdp { /** * @brief class for interacting with confifurations * diff --git a/include/fdp/objects/io_object.hxx b/include/fdp/objects/io_object.hxx index e18e214..117f18a 100644 --- a/include/fdp/objects/io_object.hxx +++ b/include/fdp/objects/io_object.hxx @@ -8,7 +8,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/objects/api_object.hxx" -namespace FDP { +namespace fdp { /** * @brief Class for API objects * diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index 299703d..57b54f2 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -29,7 +29,7 @@ #include #endif -namespace FDP { +namespace fdp { /*! ************************************************************************** * @brief calculates a hash from a given input file via SHA1 * @@ -94,6 +94,6 @@ bool file_exists( const std::string &Filename ); */ std::string read_token(const ghc::filesystem::path &token_path); -}; // namespace FDP +}; // namespace fdp #endif \ No newline at end of file diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index 84ec597..79aeba9 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -4,7 +4,7 @@ * @date 2021-05-05 * @brief File containing classes and methods for connecting to the RestAPI * - * The classes and methods within this file are used to access the FDP + * The classes and methods within this file are used to access the fdp * Data Pipeline RestAPI, they handle sending of requests and retrieval of * data as JSON strings ****************************************************************************/ @@ -27,7 +27,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace fdp { /*! ************************************************************************** * @enum RESTAPI * @brief selection of either local or remote pipeline running @@ -185,6 +185,6 @@ private: std::string url_encode(const std::string& url); -}; // namespace FDP +}; // namespace fdp #endif diff --git a/include/fdp/registry/data_io.hxx b/include/fdp/registry/data_io.hxx index 5d1bd15..6e4150c 100644 --- a/include/fdp/registry/data_io.hxx +++ b/include/fdp/registry/data_io.hxx @@ -27,7 +27,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/utilities/semver.hxx" -namespace FDP { +namespace fdp { /*! *************************************************************************** * @brief read the value of a point estimate from a given TOML file @@ -101,6 +101,6 @@ ghc::filesystem::path create_estimate(T &value, */ std::string get_first_key_(const toml::value data_table); -}; // namespace FDP +}; // namespace fdp #endif \ No newline at end of file diff --git a/include/fdp/utilities/json.hxx b/include/fdp/utilities/json.hxx index ed4e687..1cb91cd 100644 --- a/include/fdp/utilities/json.hxx +++ b/include/fdp/utilities/json.hxx @@ -12,7 +12,7 @@ #include -namespace FDP { +namespace fdp { /*! ************************************************************************** * @brief convert a JSON object to string form * @author K. Zarebski (UKAEA) @@ -29,6 +29,6 @@ namespace FDP { * ****************************************************************************/ std::string json_to_string(Json::Value &json_data); -}; // namespace FDP +}; // namespace fdp #endif \ No newline at end of file diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index a495cfb..09cb948 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -14,7 +14,7 @@ #include "spdlog/sinks/stdout_color_sinks.h" #include "spdlog/spdlog.h" -namespace FDP { +namespace fdp { class logger { @@ -28,7 +28,7 @@ namespace FDP { * */ extern const logger::sptr APILogger; -} // namespace FDP +} // namespace fdp diff --git a/include/fdp/utilities/semver.hxx b/include/fdp/utilities/semver.hxx index 27936fd..6f99abe 100644 --- a/include/fdp/utilities/semver.hxx +++ b/include/fdp/utilities/semver.hxx @@ -16,9 +16,9 @@ #include #include -namespace FDP { +namespace fdp { /*! ************************************************************************** - * @namespace FDP::Versioning + * @namespace fdp::Versioning * @brief namespace containing all semantic versioning types and classes * @author K. Zarebski (UKAEA) * @@ -199,6 +199,6 @@ private: std::string get_tag_str() const; }; }; // namespace Versioning -}; // namespace FDP +}; // namespace fdp #endif \ No newline at end of file diff --git a/src/fdp.cxx b/src/fdp.cxx index 95197b3..f19d78e 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -13,18 +13,18 @@ #include "fdp/registry/data_io.hxx" #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace fdp { /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods * * This class performs all behind the scenes operations acting as the backend to - * the user called FDP::DataPipeline class. + * the user called fdp::DataPipeline class. * This structure has been chosen to allow for unit tests to be created to test * all methods including private ones. * * @warning The class should not be used directly during implementation, - * but rather via an FDP::DataPipeline instance. + * but rather via an fdp::DataPipeline instance. * *****************************************************************************/ class DataPipeline::impl { @@ -130,17 +130,17 @@ DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, token); } -ghc::filesystem::path FDP::DataPipeline::impl::link_read(std::string &data_product){ +ghc::filesystem::path fdp::DataPipeline::impl::link_read(std::string &data_product){ return config_->link_read(data_product); } -ghc::filesystem::path FDP::DataPipeline::impl::link_write(std::string &data_product){ +ghc::filesystem::path fdp::DataPipeline::impl::link_write(std::string &data_product){ return config_->link_write(data_product); } -void FDP::DataPipeline::impl::finalise(){ +void fdp::DataPipeline::impl::finalise(){ config_->finalise(); } -std::string FDP::DataPipeline::impl::get_code_run_uuid() const { +std::string fdp::DataPipeline::impl::get_code_run_uuid() const { return config_->get_code_run_uuid(); } @@ -175,15 +175,15 @@ DataPipeline& DataPipeline::operator=(const DataPipeline& rhs) { return *this; } -ghc::filesystem::path FDP::DataPipeline::link_read(std::string &data_product){ +ghc::filesystem::path fdp::DataPipeline::link_read(std::string &data_product){ return pimpl_->link_read(data_product); } -ghc::filesystem::path FDP::DataPipeline::link_write(std::string &data_product){ +ghc::filesystem::path fdp::DataPipeline::link_write(std::string &data_product){ return pimpl_->link_write(data_product); } -void FDP::DataPipeline::finalise(){ +void fdp::DataPipeline::finalise(){ pimpl_->finalise(); } @@ -191,4 +191,4 @@ void FDP::DataPipeline::finalise(){ -}; // namespace FDP +}; // namespace fdp diff --git a/src/objects/api_object.cxx b/src/objects/api_object.cxx index f930312..ea5044f 100644 --- a/src/objects/api_object.cxx +++ b/src/objects/api_object.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/api_object.hxx" -namespace FDP { +namespace fdp { ApiObject::ApiObject() : obj_(Json::Value() ) { diff --git a/src/objects/config.cxx b/src/objects/config.cxx index d791c04..e83eba6 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/config.hxx" -namespace FDP { +namespace fdp { Config::sptr Config::construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, @@ -17,7 +17,7 @@ namespace FDP { -FDP::Config::Config(const ghc::filesystem::path &config_file_path, +fdp::Config::Config(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string &token, RESTAPI api_location) @@ -30,68 +30,68 @@ Config::~Config() { // delete api; } -YAML::Node FDP::Config::parse_yaml(ghc::filesystem::path yaml_path) { +YAML::Node fdp::Config::parse_yaml(ghc::filesystem::path yaml_path) { APILogger->debug("[Config]: Reading configuration file '{0}'", yaml_path.string().c_str()); return YAML::LoadFile(yaml_path.string().c_str()); } -YAML::Node FDP::Config::meta_data_() const { +YAML::Node fdp::Config::meta_data_() const { return config_data_["run_metadata"]; } -YAML::Node FDP::Config::config_writes_() const{ +YAML::Node fdp::Config::config_writes_() const{ return config_data_["write"]; } -YAML::Node FDP::Config::config_reads_() const{ +YAML::Node fdp::Config::config_reads_() const{ return config_data_["read"]; } -bool FDP::Config::config_has_writes() const{ +bool fdp::Config::config_has_writes() const{ if(config_data_["write"]){ return true; } return false; } -bool FDP::Config::config_has_reads() const{ +bool fdp::Config::config_has_reads() const{ if(config_data_["read"]){ return true; } return false; } -bool FDP::Config::has_writes() const{ +bool fdp::Config::has_writes() const{ return ! writes_.empty(); } -bool FDP::Config::has_reads() const{ +bool fdp::Config::has_reads() const{ return ! reads_.empty(); } -bool FDP::Config::has_inputs() const{ +bool fdp::Config::has_inputs() const{ return ! inputs_.empty(); } -bool FDP::Config::has_outputs() const{ +bool fdp::Config::has_outputs() const{ return ! outputs_.empty(); } -ghc::filesystem::path FDP::Config::get_data_store() const { +ghc::filesystem::path fdp::Config::get_data_store() const { return ghc::filesystem::path( - FDP::Config::meta_data_()["write_data_store"].as()); + fdp::Config::meta_data_()["write_data_store"].as()); } -std::string FDP::Config::get_default_input_namespace() const { +std::string fdp::Config::get_default_input_namespace() const { return meta_data_()["default_input_namespace"].as(); } -std::string FDP::Config::get_default_output_namespace() const { +std::string fdp::Config::get_default_output_namespace() const { return meta_data_()["default_output_namespace"].as(); } -void FDP::Config::validate_config(ghc::filesystem::path yaml_path, +void fdp::Config::validate_config(ghc::filesystem::path yaml_path, RESTAPI api_location) { config_data_ = parse_yaml(yaml_path); if (!config_data_["run_metadata"]) { @@ -173,7 +173,7 @@ void FDP::Config::validate_config(ghc::filesystem::path yaml_path, } -void FDP::Config::initialise(RESTAPI api_location) { +void fdp::Config::initialise(RESTAPI api_location) { // Set API URL if (api_location == RESTAPI::REMOTE) { api_url_ = API::append_with_forward_slash( @@ -429,7 +429,7 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ } -ghc::filesystem::path FDP::Config::link_read( const std::string &data_product){ +ghc::filesystem::path fdp::Config::link_read( const std::string &data_product){ YAML::Node currentRead; auto it = inputs_.find("data_product"); @@ -548,7 +548,7 @@ ghc::filesystem::path FDP::Config::link_read( const std::string &data_product){ return path_; } -void FDP::Config::finalise(){ +void fdp::Config::finalise(){ if(has_writes()){ Config::map_type::iterator it; @@ -704,4 +704,4 @@ void FDP::Config::finalise(){ } -}; // namespace FDP +}; // namespace fdp diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 236113b..0f9b835 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/metadata.hxx" -namespace FDP { +namespace fdp { std::string calculate_hash_from_file(const ghc::filesystem::path &file_path) { if (!ghc::filesystem::exists(file_path)) { throw std::invalid_argument("File '" + file_path.string() + "' not found"); @@ -85,4 +85,4 @@ std::string read_token(const ghc::filesystem::path &token_path){ return key_str_; } -}; // namespace FDP \ No newline at end of file +}; // namespace fdp \ No newline at end of file diff --git a/src/registry/api.cxx b/src/registry/api.cxx index b353408..950fa67 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -1,6 +1,6 @@ #include "fdp/registry/api.hxx" -namespace FDP { +namespace fdp { static size_t write_str_(char *ptr, size_t size, size_t nmemb, void* userdata ) { std::string* data = static_cast< std::string* >( userdata ); data->append((char *)ptr, size * nmemb); @@ -316,4 +316,4 @@ std::string API::remove_leading_forward_slash(std::string str) { return str; } -}; // namespace FDP +}; // namespace fdp diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index 1be20f1..f820f01 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -1,6 +1,6 @@ #include "fdp/registry/data_io.hxx" -namespace FDP { +namespace fdp { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { APILogger->debug("LocalFileSystem: Reading configuration file '{0}'", yaml_path.string().c_str()); @@ -38,4 +38,4 @@ double read_point_estimate_from_toml(const ghc::filesystem::path var_address) { std::string get_first_key_(const toml::value data_table) { return data_table.as_table().begin()->first; } -}; // namespace FDP +}; // namespace fdp diff --git a/src/utilities/json.cxx b/src/utilities/json.cxx index 8393d0f..73d1c52 100644 --- a/src/utilities/json.cxx +++ b/src/utilities/json.cxx @@ -1,8 +1,8 @@ #include "fdp/utilities/json.hxx" -namespace FDP { +namespace fdp { std::string json_to_string(Json::Value &json_data) { Json::StreamWriterBuilder json_str_builder_; return Json::writeString(json_str_builder_, json_data); } -} // namespace FDP \ No newline at end of file +} // namespace fdp \ No newline at end of file diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 8c775a2..0ed64a8 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,6 +1,6 @@ #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace fdp { /*! ************************************************************************** * @brief Create a the global logger pointer @@ -17,4 +17,4 @@ namespace FDP { } const logger::sptr APILogger = create_logger(); -} // namespace FDP +} // namespace fdp diff --git a/src/utilities/semver.cxx b/src/utilities/semver.cxx index c84ee15..d1adcba 100644 --- a/src/utilities/semver.cxx +++ b/src/utilities/semver.cxx @@ -1,6 +1,6 @@ #include "fdp/utilities/semver.hxx" -namespace FDP { +namespace fdp { Versioning::version::version(const std::string version_str) { const std::string delim_ = "."; const std::string meta_delim_ = "+"; @@ -83,4 +83,4 @@ std::string Versioning::version::get_tag_str() const { return ""; }; } -} // namespace FDP \ No newline at end of file +} // namespace fdp \ No newline at end of file diff --git a/test/test_api.cxx b/test/test_api.cxx index 8d82491..bda55f3 100644 --- a/test/test_api.cxx +++ b/test/test_api.cxx @@ -9,7 +9,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace fdp; class ApiTest : public ::testing::Test { protected: @@ -26,7 +26,7 @@ class ApiTest : public ::testing::Test { } ghc::filesystem::path read_csv_config = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; std::string api_url_ = "http://127.0.0.1:8000/api"; - FDP::API *api_ = new FDP::API(api_url_); + fdp::API *api_ = new fdp::API(api_url_); std::string token = read_token(ghc::filesystem::path(getHomeDirectory()) / ".fair" / "registry" / "token"); diff --git a/test/test_api_object.cxx b/test/test_api_object.cxx index d36d2f4..bc35886 100644 --- a/test/test_api_object.cxx +++ b/test/test_api_object.cxx @@ -7,7 +7,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace fdp; class ApiObjectTest : public ::testing::Test { protected: diff --git a/test/test_config.cxx b/test/test_config.cxx index a27e4cb..d893b55 100644 --- a/test/test_config.cxx +++ b/test/test_config.cxx @@ -14,7 +14,7 @@ #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace fdp; class ConfigTest : public ::testing::Test { protected: diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index d716d0e..9592bef 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -12,7 +12,7 @@ #include "gtest/gtest.h" -using namespace FDP; +using namespace fdp; class PimplTest : public ::testing::Test { protected: diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 0123a20..3ed876e 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -9,7 +9,7 @@ #include #include -using namespace FDP; +using namespace fdp; class RegistryTest : public ::testing::Test { protected: diff --git a/test/test_utilities.cxx b/test/test_utilities.cxx index 7247c6e..eb8d6cb 100644 --- a/test/test_utilities.cxx +++ b/test/test_utilities.cxx @@ -9,7 +9,7 @@ #include "json/reader.h" -using namespace FDP; +using namespace fdp; TEST(FDPAPITest, TestSemVerComparisons) { Versioning::version v1{1, 4, 5}; From 6d3d75269958768ffebaa1a27cd92dd5512fcad5 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Thu, 17 Feb 2022 14:09:25 +0000 Subject: [PATCH 06/16] Rename namespace to FairDataPipeline --- include/fdp/exceptions.hxx | 4 +-- include/fdp/fdp.hxx | 7 ++--- include/fdp/objects/api_object.hxx | 2 +- include/fdp/objects/config.hxx | 2 +- include/fdp/objects/io_object.hxx | 2 +- include/fdp/objects/metadata.hxx | 6 ++--- include/fdp/registry/api.hxx | 8 +++--- include/fdp/registry/data_io.hxx | 6 ++--- include/fdp/utilities/json.hxx | 6 ++--- include/fdp/utilities/logging.hxx | 6 ++--- include/fdp/utilities/semver.hxx | 8 +++--- src/fdp.cxx | 22 ++++++++-------- src/objects/api_object.cxx | 2 +- src/objects/config.cxx | 42 +++++++++++++++--------------- src/objects/metadata.cxx | 4 +-- src/registry/api.cxx | 4 +-- src/registry/data_io.cxx | 4 +-- src/utilities/json.cxx | 4 +-- src/utilities/logging.cxx | 4 +-- src/utilities/semver.cxx | 4 +-- test/test_api.cxx | 4 +-- test/test_api_object.cxx | 2 +- test/test_config.cxx | 2 +- test/test_pimpl.cxx | 2 +- test/test_registry.cxx | 2 +- test/test_utilities.cxx | 2 +- 26 files changed, 79 insertions(+), 82 deletions(-) diff --git a/include/fdp/exceptions.hxx b/include/fdp/exceptions.hxx index c6a6516..ff2cd60 100644 --- a/include/fdp/exceptions.hxx +++ b/include/fdp/exceptions.hxx @@ -4,7 +4,7 @@ #include #include -namespace fdp { +namespace FairDataPipeline { class config_parsing_error : public std::runtime_error { public: config_parsing_error(const std::string& message) @@ -37,6 +37,6 @@ public: write_error(const std::string& message) : std::runtime_error(message) {} }; -}; // namespace fdp +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index a172efa..796de16 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -4,9 +4,7 @@ #include "spdlog/spdlog.h" #include -//#include "fdp/registry/datapipeline.hxx" - -namespace fdp { +namespace FairDataPipeline { /** * @brief DataPipeline Class: * A PIMPL Class for interacting the the FAIR Data Pipeline @@ -72,6 +70,5 @@ namespace fdp { std::shared_ptr< DataPipeline::impl > pimpl_; }; -}; // namespace fdp - +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/objects/api_object.hxx b/include/fdp/objects/api_object.hxx index 05fa55b..6069c3c 100644 --- a/include/fdp/objects/api_object.hxx +++ b/include/fdp/objects/api_object.hxx @@ -7,7 +7,7 @@ #include "fdp/utilities/logging.hxx" -namespace fdp { +namespace FairDataPipeline { /** * @brief Class for API objects * diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index ded5fa7..e163e54 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -25,7 +25,7 @@ #include "fdp/objects/api_object.hxx" #include "fdp/objects/io_object.hxx" -namespace fdp { +namespace FairDataPipeline { /** * @brief class for interacting with confifurations * diff --git a/include/fdp/objects/io_object.hxx b/include/fdp/objects/io_object.hxx index 117f18a..0ea5314 100644 --- a/include/fdp/objects/io_object.hxx +++ b/include/fdp/objects/io_object.hxx @@ -8,7 +8,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/objects/api_object.hxx" -namespace fdp { +namespace FairDataPipeline { /** * @brief Class for API objects * diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index 57b54f2..83c43dd 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/objects/metadata.hxx + * @file FairDataPipeline/objects/metadata.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-05 * @brief File containing methods for handling of metadata @@ -29,7 +29,7 @@ #include #endif -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** * @brief calculates a hash from a given input file via SHA1 * @@ -94,6 +94,6 @@ bool file_exists( const std::string &Filename ); */ std::string read_token(const ghc::filesystem::path &token_path); -}; // namespace fdp +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index 79aeba9..b658625 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -1,10 +1,10 @@ /*! ************************************************************************** - * @file fdp/registry/api.hxx + * @file FairDataPipeline/registry/api.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-05 * @brief File containing classes and methods for connecting to the RestAPI * - * The classes and methods within this file are used to access the fdp + * The classes and methods within this file are used to access the FairDataPipeline * Data Pipeline RestAPI, they handle sending of requests and retrieval of * data as JSON strings ****************************************************************************/ @@ -27,7 +27,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** * @enum RESTAPI * @brief selection of either local or remote pipeline running @@ -185,6 +185,6 @@ private: std::string url_encode(const std::string& url); -}; // namespace fdp +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/registry/data_io.hxx b/include/fdp/registry/data_io.hxx index 6e4150c..bdf9953 100644 --- a/include/fdp/registry/data_io.hxx +++ b/include/fdp/registry/data_io.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/registry/config.hxx + * @file FairDataPipeline/registry/config.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File containing classes for interacting with the local file system @@ -27,7 +27,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/utilities/semver.hxx" -namespace fdp { +namespace FairDataPipeline { /*! *************************************************************************** * @brief read the value of a point estimate from a given TOML file @@ -101,6 +101,6 @@ ghc::filesystem::path create_estimate(T &value, */ std::string get_first_key_(const toml::value data_table); -}; // namespace fdp +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/fdp/utilities/json.hxx b/include/fdp/utilities/json.hxx index 1cb91cd..6f5e369 100644 --- a/include/fdp/utilities/json.hxx +++ b/include/fdp/utilities/json.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/json.hxx + * @file FairDataPipeline/utilities/json.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains methods relating to handling of JSON based data @@ -12,7 +12,7 @@ #include -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** * @brief convert a JSON object to string form * @author K. Zarebski (UKAEA) @@ -29,6 +29,6 @@ namespace fdp { * ****************************************************************************/ std::string json_to_string(Json::Value &json_data); -}; // namespace fdp +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index 09cb948..67118c9 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/logging.hxx + * @file FairDataPipeline/utilities/logging.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains pointers for creating global logger during session @@ -14,7 +14,7 @@ #include "spdlog/sinks/stdout_color_sinks.h" #include "spdlog/spdlog.h" -namespace fdp { +namespace FairDataPipeline { class logger { @@ -28,7 +28,7 @@ namespace fdp { * */ extern const logger::sptr APILogger; -} // namespace fdp +} // namespace FairDataPipeline diff --git a/include/fdp/utilities/semver.hxx b/include/fdp/utilities/semver.hxx index 6f99abe..8b2e904 100644 --- a/include/fdp/utilities/semver.hxx +++ b/include/fdp/utilities/semver.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/semver.hxx + * @file FairDataPipeline/utilities/semver.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains classes and types for validating semantic versioning @@ -16,9 +16,9 @@ #include #include -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** - * @namespace fdp::Versioning + * @namespace FairDataPipeline::Versioning * @brief namespace containing all semantic versioning types and classes * @author K. Zarebski (UKAEA) * @@ -199,6 +199,6 @@ private: std::string get_tag_str() const; }; }; // namespace Versioning -}; // namespace fdp +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/src/fdp.cxx b/src/fdp.cxx index f19d78e..19189ae 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -13,18 +13,18 @@ #include "fdp/registry/data_io.hxx" #include "fdp/utilities/logging.hxx" -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods * * This class performs all behind the scenes operations acting as the backend to - * the user called fdp::DataPipeline class. + * the user called FairDataPipeline::DataPipeline class. * This structure has been chosen to allow for unit tests to be created to test * all methods including private ones. * * @warning The class should not be used directly during implementation, - * but rather via an fdp::DataPipeline instance. + * but rather via an FairDataPipeline::DataPipeline instance. * *****************************************************************************/ class DataPipeline::impl { @@ -130,17 +130,17 @@ DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, token); } -ghc::filesystem::path fdp::DataPipeline::impl::link_read(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_read(std::string &data_product){ return config_->link_read(data_product); } -ghc::filesystem::path fdp::DataPipeline::impl::link_write(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_write(std::string &data_product){ return config_->link_write(data_product); } -void fdp::DataPipeline::impl::finalise(){ +void FairDataPipeline::DataPipeline::impl::finalise(){ config_->finalise(); } -std::string fdp::DataPipeline::impl::get_code_run_uuid() const { +std::string FairDataPipeline::DataPipeline::impl::get_code_run_uuid() const { return config_->get_code_run_uuid(); } @@ -175,15 +175,15 @@ DataPipeline& DataPipeline::operator=(const DataPipeline& rhs) { return *this; } -ghc::filesystem::path fdp::DataPipeline::link_read(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::link_read(std::string &data_product){ return pimpl_->link_read(data_product); } -ghc::filesystem::path fdp::DataPipeline::link_write(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::link_write(std::string &data_product){ return pimpl_->link_write(data_product); } -void fdp::DataPipeline::finalise(){ +void FairDataPipeline::DataPipeline::finalise(){ pimpl_->finalise(); } @@ -191,4 +191,4 @@ void fdp::DataPipeline::finalise(){ -}; // namespace fdp +}; // namespace FairDataPipeline diff --git a/src/objects/api_object.cxx b/src/objects/api_object.cxx index ea5044f..eba0606 100644 --- a/src/objects/api_object.cxx +++ b/src/objects/api_object.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/api_object.hxx" -namespace fdp { +namespace FairDataPipeline { ApiObject::ApiObject() : obj_(Json::Value() ) { diff --git a/src/objects/config.cxx b/src/objects/config.cxx index e83eba6..33c18ef 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/config.hxx" -namespace fdp { +namespace FairDataPipeline { Config::sptr Config::construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, @@ -17,7 +17,7 @@ namespace fdp { -fdp::Config::Config(const ghc::filesystem::path &config_file_path, +FairDataPipeline::Config::Config(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string &token, RESTAPI api_location) @@ -30,68 +30,68 @@ Config::~Config() { // delete api; } -YAML::Node fdp::Config::parse_yaml(ghc::filesystem::path yaml_path) { +YAML::Node FairDataPipeline::Config::parse_yaml(ghc::filesystem::path yaml_path) { APILogger->debug("[Config]: Reading configuration file '{0}'", yaml_path.string().c_str()); return YAML::LoadFile(yaml_path.string().c_str()); } -YAML::Node fdp::Config::meta_data_() const { +YAML::Node FairDataPipeline::Config::meta_data_() const { return config_data_["run_metadata"]; } -YAML::Node fdp::Config::config_writes_() const{ +YAML::Node FairDataPipeline::Config::config_writes_() const{ return config_data_["write"]; } -YAML::Node fdp::Config::config_reads_() const{ +YAML::Node FairDataPipeline::Config::config_reads_() const{ return config_data_["read"]; } -bool fdp::Config::config_has_writes() const{ +bool FairDataPipeline::Config::config_has_writes() const{ if(config_data_["write"]){ return true; } return false; } -bool fdp::Config::config_has_reads() const{ +bool FairDataPipeline::Config::config_has_reads() const{ if(config_data_["read"]){ return true; } return false; } -bool fdp::Config::has_writes() const{ +bool FairDataPipeline::Config::has_writes() const{ return ! writes_.empty(); } -bool fdp::Config::has_reads() const{ +bool FairDataPipeline::Config::has_reads() const{ return ! reads_.empty(); } -bool fdp::Config::has_inputs() const{ +bool FairDataPipeline::Config::has_inputs() const{ return ! inputs_.empty(); } -bool fdp::Config::has_outputs() const{ +bool FairDataPipeline::Config::has_outputs() const{ return ! outputs_.empty(); } -ghc::filesystem::path fdp::Config::get_data_store() const { +ghc::filesystem::path FairDataPipeline::Config::get_data_store() const { return ghc::filesystem::path( - fdp::Config::meta_data_()["write_data_store"].as()); + FairDataPipeline::Config::meta_data_()["write_data_store"].as()); } -std::string fdp::Config::get_default_input_namespace() const { +std::string FairDataPipeline::Config::get_default_input_namespace() const { return meta_data_()["default_input_namespace"].as(); } -std::string fdp::Config::get_default_output_namespace() const { +std::string FairDataPipeline::Config::get_default_output_namespace() const { return meta_data_()["default_output_namespace"].as(); } -void fdp::Config::validate_config(ghc::filesystem::path yaml_path, +void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, RESTAPI api_location) { config_data_ = parse_yaml(yaml_path); if (!config_data_["run_metadata"]) { @@ -173,7 +173,7 @@ void fdp::Config::validate_config(ghc::filesystem::path yaml_path, } -void fdp::Config::initialise(RESTAPI api_location) { +void FairDataPipeline::Config::initialise(RESTAPI api_location) { // Set API URL if (api_location == RESTAPI::REMOTE) { api_url_ = API::append_with_forward_slash( @@ -429,7 +429,7 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ } -ghc::filesystem::path fdp::Config::link_read( const std::string &data_product){ +ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &data_product){ YAML::Node currentRead; auto it = inputs_.find("data_product"); @@ -548,7 +548,7 @@ ghc::filesystem::path fdp::Config::link_read( const std::string &data_product){ return path_; } -void fdp::Config::finalise(){ +void FairDataPipeline::Config::finalise(){ if(has_writes()){ Config::map_type::iterator it; @@ -704,4 +704,4 @@ void fdp::Config::finalise(){ } -}; // namespace fdp +}; // namespace FairDataPipeline diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 0f9b835..05ce8ed 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/metadata.hxx" -namespace fdp { +namespace FairDataPipeline { std::string calculate_hash_from_file(const ghc::filesystem::path &file_path) { if (!ghc::filesystem::exists(file_path)) { throw std::invalid_argument("File '" + file_path.string() + "' not found"); @@ -85,4 +85,4 @@ std::string read_token(const ghc::filesystem::path &token_path){ return key_str_; } -}; // namespace fdp \ No newline at end of file +}; // namespace FairDataPipeline \ No newline at end of file diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 950fa67..745fedc 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -1,6 +1,6 @@ #include "fdp/registry/api.hxx" -namespace fdp { +namespace FairDataPipeline { static size_t write_str_(char *ptr, size_t size, size_t nmemb, void* userdata ) { std::string* data = static_cast< std::string* >( userdata ); data->append((char *)ptr, size * nmemb); @@ -316,4 +316,4 @@ std::string API::remove_leading_forward_slash(std::string str) { return str; } -}; // namespace fdp +}; // namespace FairDataPipeline diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index f820f01..6116ac6 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -1,6 +1,6 @@ #include "fdp/registry/data_io.hxx" -namespace fdp { +namespace FairDataPipeline { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { APILogger->debug("LocalFileSystem: Reading configuration file '{0}'", yaml_path.string().c_str()); @@ -38,4 +38,4 @@ double read_point_estimate_from_toml(const ghc::filesystem::path var_address) { std::string get_first_key_(const toml::value data_table) { return data_table.as_table().begin()->first; } -}; // namespace fdp +}; // namespace FairDataPipeline diff --git a/src/utilities/json.cxx b/src/utilities/json.cxx index 73d1c52..e0b944b 100644 --- a/src/utilities/json.cxx +++ b/src/utilities/json.cxx @@ -1,8 +1,8 @@ #include "fdp/utilities/json.hxx" -namespace fdp { +namespace FairDataPipeline { std::string json_to_string(Json::Value &json_data) { Json::StreamWriterBuilder json_str_builder_; return Json::writeString(json_str_builder_, json_data); } -} // namespace fdp \ No newline at end of file +} // namespace FairDataPipeline \ No newline at end of file diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 0ed64a8..9f7bd98 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,6 +1,6 @@ #include "fdp/utilities/logging.hxx" -namespace fdp { +namespace FairDataPipeline { /*! ************************************************************************** * @brief Create a the global logger pointer @@ -17,4 +17,4 @@ namespace fdp { } const logger::sptr APILogger = create_logger(); -} // namespace fdp +} // namespace FairDataPipeline diff --git a/src/utilities/semver.cxx b/src/utilities/semver.cxx index d1adcba..4db3ca4 100644 --- a/src/utilities/semver.cxx +++ b/src/utilities/semver.cxx @@ -1,6 +1,6 @@ #include "fdp/utilities/semver.hxx" -namespace fdp { +namespace FairDataPipeline { Versioning::version::version(const std::string version_str) { const std::string delim_ = "."; const std::string meta_delim_ = "+"; @@ -83,4 +83,4 @@ std::string Versioning::version::get_tag_str() const { return ""; }; } -} // namespace fdp \ No newline at end of file +} // namespace FairDataPipeline \ No newline at end of file diff --git a/test/test_api.cxx b/test/test_api.cxx index bda55f3..50ccd38 100644 --- a/test/test_api.cxx +++ b/test/test_api.cxx @@ -9,7 +9,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace fdp; +using namespace FairDataPipeline; class ApiTest : public ::testing::Test { protected: @@ -26,7 +26,7 @@ class ApiTest : public ::testing::Test { } ghc::filesystem::path read_csv_config = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; std::string api_url_ = "http://127.0.0.1:8000/api"; - fdp::API *api_ = new fdp::API(api_url_); + FairDataPipeline::API *api_ = new FairDataPipeline::API(api_url_); std::string token = read_token(ghc::filesystem::path(getHomeDirectory()) / ".fair" / "registry" / "token"); diff --git a/test/test_api_object.cxx b/test/test_api_object.cxx index bc35886..884a317 100644 --- a/test/test_api_object.cxx +++ b/test/test_api_object.cxx @@ -7,7 +7,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace fdp; +using namespace FairDataPipeline; class ApiObjectTest : public ::testing::Test { protected: diff --git a/test/test_config.cxx b/test/test_config.cxx index d893b55..a38335a 100644 --- a/test/test_config.cxx +++ b/test/test_config.cxx @@ -14,7 +14,7 @@ #include "fdp/utilities/logging.hxx" -using namespace fdp; +using namespace FairDataPipeline; class ConfigTest : public ::testing::Test { protected: diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index 9592bef..7bdd5bf 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -12,7 +12,7 @@ #include "gtest/gtest.h" -using namespace fdp; +using namespace FairDataPipeline; class PimplTest : public ::testing::Test { protected: diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 3ed876e..9b4caac 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -9,7 +9,7 @@ #include #include -using namespace fdp; +using namespace FairDataPipeline; class RegistryTest : public ::testing::Test { protected: diff --git a/test/test_utilities.cxx b/test/test_utilities.cxx index eb8d6cb..01a0004 100644 --- a/test/test_utilities.cxx +++ b/test/test_utilities.cxx @@ -9,7 +9,7 @@ #include "json/reader.h" -using namespace fdp; +using namespace FairDataPipeline; TEST(FDPAPITest, TestSemVerComparisons) { Versioning::version v1{1, 4, 5}; From 0b2ab2ac5a6f045b91d8d386474b5f52f3dfd088 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Wed, 30 Mar 2022 00:15:05 +0100 Subject: [PATCH 07/16] New logging api --- include/fdp/fdp.hxx | 25 +++-- include/fdp/registry/api.hxx | 7 +- include/fdp/registry/data_io.hxx | 7 +- include/fdp/utilities/logging.hxx | 170 ++++++++++++++++++++++++++++-- src/fdp.cxx | 105 ++++++++++++------ src/objects/api_object.cxx | 2 +- src/objects/config.cxx | 89 ++++++++-------- src/registry/api.cxx | 27 +++-- src/registry/data_io.cxx | 2 +- src/utilities/logging.cxx | 96 ++++++++++++++--- test/test_api.cxx | 7 +- test/test_config.cxx | 50 +++++---- test/test_pimpl.cxx | 14 +-- test/test_registry.cxx | 22 ++-- 14 files changed, 464 insertions(+), 159 deletions(-) diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 796de16..03a7269 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -1,7 +1,8 @@ #ifndef __FDP__ #define __FDP__ -#include "spdlog/spdlog.h" +//#include "spdlog/spdlog.h" +#include "utilities/logging.hxx" #include namespace FairDataPipeline { @@ -13,6 +14,7 @@ namespace FairDataPipeline { class DataPipeline { public: + typedef std::shared_ptr< DataPipeline > sptr; /** * @brief Construct a new Data Pipeline (PIMPL) * @@ -21,11 +23,12 @@ namespace FairDataPipeline { * @param token * @param log_level */ - explicit DataPipeline( + static sptr construct( const std::string &config_file_path, const std::string &script_file_path, std::string token = "", - spdlog::level::level_enum log_level = spdlog::level::info); + logger::LOG_LEVEL log_level = logger::LOG_LEVEL::log_level_info ); + /** * @brief Destroy the Data Pipeline object @@ -33,11 +36,9 @@ namespace FairDataPipeline { */ ~DataPipeline(); - DataPipeline(DataPipeline &&rhs) noexcept; - DataPipeline &operator=(DataPipeline &&rhs) noexcept; + //DataPipeline(DataPipeline &&rhs) noexcept; + //DataPipeline &operator=(DataPipeline &&rhs) noexcept; - DataPipeline(const DataPipeline &rhs); - DataPipeline &operator=(const DataPipeline &rhs); /** * @brief Return a path to a given data product @@ -66,6 +67,16 @@ namespace FairDataPipeline { void finalise(); private: + explicit DataPipeline( + const std::string &config_file_path, + const std::string &script_file_path, + std::string token = "", + logger::LOG_LEVEL log_level = logger::LOG_LEVEL::log_level_info); + + DataPipeline(const DataPipeline &rhs) = delete; + + DataPipeline &operator=(const DataPipeline &rhs) = delete; + class impl; std::shared_ptr< DataPipeline::impl > pimpl_; diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index b658625..94692a5 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -72,8 +72,8 @@ public: * * @param url_root the root of the query address, e.g. localhost:8000/api ***************************************************************************/ - API( const std::string& url_root) - : url_root_(API::append_with_forward_slash(url_root)) {} + static sptr construct( const std::string& url_root ); + /** * @brief sends the given 'packet' of information to the RestAPI @@ -166,6 +166,9 @@ public: static std::string remove_leading_forward_slash(std::string str); private: + API( const std::string& url_root) + : url_root_(API::append_with_forward_slash(url_root)) {} + std::string url_root_; CURL *setup_json_session_(std::string &addr_path, std::string *response, long &http_code, std::string token = ""); diff --git a/include/fdp/registry/data_io.hxx b/include/fdp/registry/data_io.hxx index bdf9953..884409d 100644 --- a/include/fdp/registry/data_io.hxx +++ b/include/fdp/registry/data_io.hxx @@ -87,8 +87,9 @@ ghc::filesystem::path create_estimate(T &value, toml_out_.close(); - APILogger->debug("FileSystem:CreateEstimate: Wrote point estimate to '{0}'", - output_filename_.string()); + + logger::sptr the_logger = logger::get_logger(); + the_logger->debug( "FileSystem:CreateEstimate: Wrote point estimate to '{0}'", output_filename_.string()); return output_filename_; } @@ -103,4 +104,4 @@ std::string get_first_key_(const toml::value data_table); }; // namespace FairDataPipeline -#endif \ No newline at end of file +#endif diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index 67118c9..bee2871 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -11,23 +11,179 @@ #ifndef __FDP_LOGGING_HXX__ #define __FDP_LOGGING_HXX__ -#include "spdlog/sinks/stdout_color_sinks.h" +#include +#include +#include + #include "spdlog/spdlog.h" +#if 0 +#define TRACE 0 +#define DEBUG 1 +#define INFO 2 +#define WARN 3 +#define ERROR 4 +#define CRITICAL 5 +#define OFF 6 +#endif + namespace FairDataPipeline { + namespace logging { + + enum LOG_LEVEL + { + TRACE = 0, + DEBUG = 1, + INFO = 2, + WARN = 3, + ERROR = 4, + CRITICAL = 5, + OFF = 6 + }; + + struct ISink + { + virtual int log( enum LOG_LEVEL msg_lvl, const std::string& s) = 0; + virtual ~ISink(){}; + }; + + class Sink : public ISink + { + public: + Sink( enum LOG_LEVEL log_lvl ) : ISink(), _log_lvl(log_lvl) {} + + virtual int log( enum LOG_LEVEL msg_lvl, const std::string& s){return 1;} + + template< typename T > + int execute( enum LOG_LEVEL msg_lvl, const T& s) + { + if( this->should_log( msg_lvl ) ) + { + std::ostringstream oss; + oss << s; + + this->log( msg_lvl, oss.str() ); + } + return 0; + } + + enum LOG_LEVEL log_level(){ return _log_lvl;} + void log_level( enum LOG_LEVEL log_lvl ){ _log_lvl = log_lvl;} + + bool should_log( enum LOG_LEVEL msg_lvl ){ return msg_lvl >= this->log_level();} + + private: + + enum LOG_LEVEL _log_lvl; + }; + + + class OStreamSink : public Sink + { + public: + OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os){ ;} + + int log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + if( this->should_log( msg_lvl ) ) + _os << s; + + return 0; + } + + private: + std::ostream& _os; + }; + + class CompositeSink : public Sink + { + public: + CompositeSink(enum LOG_LEVEL log_lvl) : Sink(log_lvl){ ;} + + int log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + if( this->should_log( msg_lvl ) ) + { + for( int i = 0; i < _sinks.size(); ++i ) + { + Sink* sink = _sinks[i]; + sink->log( msg_lvl, s ); + } + } + + return 0; + } + + void add_sink( Sink* sink ){ _sinks.push_back( sink ); } + + private: + std::vector< Sink* > _sinks; + }; + + + + class MsgLogger + { + public: + MsgLogger( enum LOG_LEVEL msg_lvl, Sink& sink ); + + template< typename T > + MsgLogger& operator<<( const T& s) + { + this->_sink.execute( _msg_lvl, s ); + return *this; + } + + private: + enum LOG_LEVEL _msg_lvl; + Sink& _sink; + }; + + class Logger + { + public: + + Logger( enum LOG_LEVEL lvl, Sink& sink ) + : _sink(sink) + { + _log_lvl = lvl; + } + + MsgLogger info(); + MsgLogger debug(); + MsgLogger trace(); + MsgLogger warn(); + MsgLogger critical(); + MsgLogger error(); + + private: + + enum LOG_LEVEL _log_lvl; + Sink& _sink; + }; + } class logger { public: + enum LOG_LEVEL + { + log_level_trace = 0, + log_level_debug = 1, + log_level_info = 2, + log_level_warn = 3, + log_level_error = 4, + log_level_critical = 5, + log_level_off = 6 + }; + typedef std::shared_ptr sptr; + + static sptr get_logger(); + private: + static sptr _instance; }; - /** - * @brief Global logger used across all FDP scripts - * @author K. Zarebski (UKAEA) - * - */ - extern const logger::sptr APILogger; } // namespace FairDataPipeline diff --git a/src/fdp.cxx b/src/fdp.cxx index 19189ae..ad4c20f 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -14,6 +14,20 @@ #include "fdp/utilities/logging.hxx" namespace FairDataPipeline { + spdlog::level::level_enum LOG_LEVEL_2_spdlog( logger::LOG_LEVEL fdp_log_level ) + { + std::map< logger::LOG_LEVEL, spdlog::level::level_enum > m; + + m[ logger::LOG_LEVEL::log_level_trace ] = spdlog::level::trace; + m[ logger::LOG_LEVEL::log_level_debug ] = spdlog::level::debug; + m[ logger::LOG_LEVEL::log_level_info ] = spdlog::level::info; + m[ logger::LOG_LEVEL::log_level_warn ] = spdlog::level::warn; + m[ logger::LOG_LEVEL::log_level_error ] = spdlog::level::err; + m[ logger::LOG_LEVEL::log_level_critical ] = spdlog::level::critical; + m[ logger::LOG_LEVEL::log_level_off ] = spdlog::level::off; + + return m[ fdp_log_level ]; + } /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods @@ -37,6 +51,13 @@ namespace FairDataPipeline { std::string token_() const {return config_->get_token();} RESTAPI api_location_() const {return config_->get_rest_api_location();} + impl(const ghc::filesystem::path &config_file_path, + const ghc::filesystem::path &file_system_path, + const std::string& token, + logger::LOG_LEVEL log_lvl, + RESTAPI api_location = RESTAPI::LOCAL); + + impl(const impl &dp) = delete; impl& operator=(const impl& ) = delete; public: @@ -50,13 +71,13 @@ namespace FairDataPipeline { * @param log_level level for the output logging statements * @param api_location whether to use local/remote RestAPI endpoint ***************************************************************************/ - impl(const ghc::filesystem::path &config_file_path, + static sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - spdlog::level::level_enum log_level = spdlog::level::info, + logger::LOG_LEVEL log_level, RESTAPI api_location = RESTAPI::LOCAL); - impl(const impl &dp) = default; + /** * @brief Default Destructor @@ -100,34 +121,55 @@ namespace FairDataPipeline { std::string get_code_run_uuid() const; }; -#if 0 -DataPipeline::impl::impl(const impl &dp) +DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::path &config_file_path, + const ghc::filesystem::path &script_file_path, + const std::string& token, + logger::LOG_LEVEL log_level, + RESTAPI api_location) { - impl( dp.config_file_path_() - , dp.script_file_path_() - , token_(), spdlog::level::info - , api_location_() - ); + + sptr pobj = impl::sptr( new impl( config_file_path, + script_file_path, + token, + log_level, + api_location ) ); + return pobj; } -#endif DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - spdlog::level::level_enum log_level, + logger::LOG_LEVEL log_level, RESTAPI api_location) { + spdlog::level::level_enum spd_log_lvl = LOG_LEVEL_2_spdlog( log_level ); + this->config_ = Config::construct(config_file_path, script_file_path, token, api_location); const std::string api_root_ = config_->get_api_url(); + +#if 0 spdlog::set_default_logger(APILogger); - APILogger->set_level(log_level); + APILogger->set_level( spd_log_lvl ); APILogger->info("\n[Configuration]\n\t- Config Path: {0}\n\t- API Root: " "{1}\n\t- FDP API Token: {2}", config_file_path.string(), api_root_, token); +#else + + logger::sptr the_logger = logger::get_logger(); + + //spdlog::set_default_logger(the_logger); + the_logger->set_level( spd_log_lvl ); + the_logger->info("\n[Configuration]\n\t- Config Path: {0}\n\t- API Root: " + "{1}\n\t- FDP API Token: {2}", + config_file_path.string(), api_root_, + token); + +#endif + } ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_read(std::string &data_product){ @@ -144,35 +186,32 @@ std::string FairDataPipeline::DataPipeline::impl::get_code_run_uuid() const { return config_->get_code_run_uuid(); } - - - - DataPipeline::~DataPipeline() = default; -DataPipeline::DataPipeline(DataPipeline &&) noexcept = default; -DataPipeline& DataPipeline::operator=(DataPipeline &&) noexcept = default; - -DataPipeline::DataPipeline(const DataPipeline& rhs) - : pimpl_(std::make_shared< DataPipeline::impl >(*rhs.pimpl_)) -{} -DataPipeline::DataPipeline( +DataPipeline::sptr DataPipeline::construct( const std::string &config_file_path, const std::string &script_file_path, std::string token, - spdlog::level::level_enum log_level ) -: pimpl_(std::make_shared< DataPipeline::impl >(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token, - log_level)) + logger::LOG_LEVEL log_level ) { - APILogger->debug("DataPipeline: Initialising session '{0}'", pimpl_->get_code_run_uuid()); + return DataPipeline::sptr( new DataPipeline( + config_file_path, + script_file_path, + token, + log_level ) ); } -DataPipeline& DataPipeline::operator=(const DataPipeline& rhs) { - if (this != &rhs) - pimpl_.reset(new impl(*rhs.pimpl_)); - - return *this; +DataPipeline::DataPipeline( + const std::string &config_file_path, + const std::string &script_file_path, + std::string token, + logger::LOG_LEVEL log_level ) +: pimpl_( DataPipeline::impl::construct(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token, + log_level)) +{ + logger::sptr logger = logger::get_logger(); + logger->debug("DataPipeline: Initialising session '{0}'", pimpl_->get_code_run_uuid()); } ghc::filesystem::path FairDataPipeline::DataPipeline::link_read(std::string &data_product){ diff --git a/src/objects/api_object.cxx b/src/objects/api_object.cxx index eba0606..e488a23 100644 --- a/src/objects/api_object.cxx +++ b/src/objects/api_object.cxx @@ -16,7 +16,7 @@ namespace FairDataPipeline { ApiObject::sptr ApiObject::construct(void) { - ApiObject::sptr pobj = std::shared_ptr< ApiObject >( new ApiObject() ); + ApiObject::sptr pobj = ApiObject::sptr( new ApiObject() ); return pobj; } diff --git a/src/objects/config.cxx b/src/objects/config.cxx index 33c18ef..2722b60 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -7,7 +7,7 @@ namespace FairDataPipeline { const std::string &token, RESTAPI api_location) { - Config::sptr pobj = std::shared_ptr< Config >( new Config( + Config::sptr pobj = Config::sptr( new Config( config_file_path , script_file_path , token @@ -25,13 +25,14 @@ FairDataPipeline::Config::Config(const ghc::filesystem::path &config_file_path, rest_api_location_(api_location) { validate_config(config_file_path, api_location); initialise(api_location); -} + + } + Config::~Config() { - // delete api; } YAML::Node FairDataPipeline::Config::parse_yaml(ghc::filesystem::path yaml_path) { - APILogger->debug("[Config]: Reading configuration file '{0}'", + logger::get_logger()->debug("[Config]: Reading configuration file '{0}'", yaml_path.string().c_str()); return YAML::LoadFile(yaml_path.string().c_str()); } @@ -95,7 +96,7 @@ void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, RESTAPI api_location) { config_data_ = parse_yaml(yaml_path); if (!config_data_["run_metadata"]) { - APILogger->error( + logger::get_logger()->error( "Failed to obtain run metadata, the key 'run_metadata' is not" " present in the configuration"); throw config_parsing_error("No run metadata provided"); @@ -103,7 +104,7 @@ void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, if (api_location == RESTAPI::LOCAL && !meta_data_()["local_data_registry_url"]) { - APILogger->error("Could not determine URL for local repository, key " + logger::get_logger()->error("Could not determine URL for local repository, key " "'local_data_registry_url' " "is absent from configuration"); throw config_parsing_error("Failed to read local repository url"); @@ -111,7 +112,7 @@ void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, else if (api_location == RESTAPI::REMOTE && !meta_data_()["remote_data_registry_url"]) { - APILogger->error("Could not determine URL for remote repository, key " + logger::get_logger()->error("Could not determine URL for remote repository, key " "'remote_data_registry_url' " "is absent from configuration"); throw config_parsing_error("Failed to read remote repository url"); @@ -119,40 +120,40 @@ void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, else if (!(api_location == RESTAPI::LOCAL || api_location == RESTAPI::REMOTE)) { - APILogger->error("Unrecognised API location"); + logger::get_logger()->error("Unrecognised API location"); throw config_parsing_error("Failed to resolve registry location"); } if (!meta_data_()["write_data_store"]) { - APILogger->error("Failed to Read: [\"write_data_store\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"write_data_store\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"write_data_store\"] from " + yaml_path.string()); } if (!meta_data_()["default_input_namespace"]) { - APILogger->error("Failed to Read: [\"default_input_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"default_input_namespace\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"default_input_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["default_output_namespace"]) { - APILogger->error("Failed to Read: [\"default_output_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"default_output_namespace\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"default_output_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["latest_commit"]) { - APILogger->error("Failed to Read: [\"latest_commit\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"latest_commit\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"latest_commit\"] from " + yaml_path.string()); } if (!meta_data_()["remote_repo"]) { - APILogger->error("Failed to Read: [\"remote_repo\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"remote_repo\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"remote_repo\"] from " + yaml_path.string()); } if (!meta_data_()["description"]) { - APILogger->error("Failed to Read: [\"description\"] from {0}", yaml_path.string()); + logger::get_logger()->error("Failed to Read: [\"description\"] from {0}", yaml_path.string()); throw config_parsing_error("Failed to Read: [\"description\"] from " + yaml_path.string()); } if (!file_exists(script_file_path_.string())) { - APILogger->error("Submission script: {0} does not exist", script_file_path_.string()); + logger::get_logger()->error("Submission script: {0} does not exist", script_file_path_.string()); throw std::runtime_error("Submission script: " + script_file_path_.string() + " does not exist"); } @@ -183,10 +184,10 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { meta_data_()["local_data_registry_url"].as()); } - APILogger->info("Reading {0} from local filestore", + logger::get_logger()->info("Reading {0} from local filestore", config_file_path_.string()); // Create and API object as a shared pointer - api_ = std::make_shared(api_url_); + api_ = API::construct(api_url_); // Get the admin user from registry Json::Value user_json_; @@ -196,7 +197,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->user_ = ApiObject::from_json( j [0]); if (user_->is_empty()) { - APILogger->error("User: Admin Not Found"); + logger::get_logger()->error("User: Admin Not Found"); throw std::runtime_error("User: Admin Not Found"); } @@ -210,7 +211,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->author_ = ApiObject::from_json( j_author ); if (author_->is_empty()) { - APILogger->error( + logger::get_logger()->error( "Author for User Admin not found please ensure you have run fair init"); throw std::runtime_error( "Author Not Found: Please ensure you have run fair init"); @@ -257,7 +258,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { config_value_["file_type"] = config_file_type_url_; - APILogger->info("Writing config file {0} to registry", config_file_path_.string()); + logger::get_logger()->info("Writing config file {0} to registry", config_file_path_.string()); Json::Value j_config_obj = api_->post("object", config_value_, token_); @@ -295,7 +296,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { script_value_["filetype"] = script_file_type_url_.asString(); script_value_["storage_location"] = script_storage_location_->get_uri(); - APILogger->info("Writing script file {0} to registry", script_file_path_.string()); + logger::get_logger()->info("Writing script file {0} to registry", script_file_path_.string()); Json::Value j_script_obj = api_->post("object", script_value_, token_); this->script_obj_ = ApiObject::from_json( j_script_obj ); @@ -335,12 +336,12 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { code_run_value_["input_urls"] = Json::arrayValue; code_run_value_["output_urls"] = Json::arrayValue; - APILogger->info("Writing new code run to registry"); + logger::get_logger()->info("Writing new code run to registry"); Json::Value j_code_run = api_->post("code_run", code_run_value_, token_); this->code_run_ = ApiObject::from_json( j_code_run ); - APILogger->info("Code run {0} successfully generated", code_run_->get_value_as_string("uuid")); + logger::get_logger()->info("Code run {0} successfully generated", code_run_->get_value_as_string("uuid")); } std::string Config::get_config_directory() const{ @@ -354,11 +355,11 @@ std::string Config::get_code_run_uuid() const{ ghc::filesystem::path Config::link_write( const std::string& data_product){ if (!config_has_writes()){ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } - //APILogger->info("Node Type: {0}", config_writes_().Type()); + //logger::get_logger()->info("Node Type: {0}", config_writes_().Type()); YAML::Node currentWrite; @@ -374,30 +375,30 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ } } else{ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentWrite) { - APILogger->error("Config Error: Cannot Find {0} in writes", data_product); + logger::get_logger()->error("Config Error: Cannot Find {0} in writes", data_product); throw config_parsing_error("Config Error: cannot find " + data_product + "in writes"); } if(!currentWrite["description"]) { - APILogger->error("Config Error: Cannot Find description of {0} in writes", data_product); + logger::get_logger()->error("Config Error: Cannot Find description of {0} in writes", data_product); throw config_parsing_error("Config Error: cannot find description of " + data_product + "in writes"); } if(!currentWrite["file_type"]) { - APILogger->error("Config Error: Cannot Find file_type of {0} in writes", data_product); + logger::get_logger()->error("Config Error: Cannot Find file_type of {0} in writes", data_product); throw config_parsing_error("Config Error: cannot find file_type of " + data_product + "in writes"); } if(!currentWrite["use"]["version"]){ - APILogger->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); currentWrite["use"]["version"] = "0.0.1"; } @@ -412,7 +413,7 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ std::string filename_("dat-" + generate_random_hash() + "." + currentWrite["file_type"].as()); ghc::filesystem::path path_ = ghc::filesystem::path(meta_data_()["write_data_store"].as()) / currentWrite["use"]["namespace"].as() / currentWrite["use"]["data_product"].as() / filename_; - APILogger->info("Link Path: {0}", path_.string()); + logger::get_logger()->info("Link Path: {0}", path_.string()); // Create Directory ghc::filesystem::create_directories(path_.parent_path().string()); @@ -449,18 +450,18 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da } } else{ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentRead) { - APILogger->error("Config Error: Cannot Find {0} in reads", data_product); + logger::get_logger()->error("Config Error: Cannot Find {0} in reads", data_product); throw config_parsing_error("Config Error: cannot find " + data_product + "in reads"); } if(!currentRead["use"]["version"]){ - APILogger->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); currentRead["use"]["version"] = "0.0.1"; } @@ -479,7 +480,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr namespaceObj = ApiObject::from_json( j_namespace ); if (namespaceObj->is_empty()){ - APILogger->error("Namespace Error: could not find namespace {0} in registry", currentRead["use"]["namespace"].as()); + logger::get_logger()->error("Namespace Error: could not find namespace {0} in registry", currentRead["use"]["namespace"].as()); throw std::runtime_error("Namespace Error: could not find namespace " + currentRead["use"]["namespace"].as() + " in Registry"); } @@ -492,7 +493,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr dataProductObj = ApiObject::from_json( j_data_prod_obj ); //ApiObject dataProductObj = ApiObject(api_->get_by_json_query("data_product", dataProductData)[0]); if (dataProductObj->is_empty()){ - APILogger->error("data_product Error: could not find data_product {0} in registry", currentRead["use"]["data_product"].as()); + logger::get_logger()->error("data_product Error: could not find data_product {0} in registry", currentRead["use"]["data_product"].as()); throw std::runtime_error("Namespace Error: could not find data_product " + currentRead["use"]["data_product"].as() + " in Registry"); } @@ -500,7 +501,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr obj = ApiObject::from_json( _j_ ); if (obj->is_empty()){ - APILogger->error("data_product Error: could not find data_product object {0} in registry", dataProductObj->get_value_as_string("object")); + logger::get_logger()->error("data_product Error: could not find data_product object {0} in registry", dataProductObj->get_value_as_string("object")); throw std::runtime_error("Namespace Error: could not find data_product object " + dataProductObj->get_value_as_string("object") + " in Registry"); } @@ -510,7 +511,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr componentObj = ApiObject::from_json( j_obj_component ); if (componentObj->is_empty()){ - APILogger->error("data_product object Error: could not find data_product component_object for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error("data_product object Error: could not find data_product component_object for {0} in registry", obj->get_value_as_string("object")); throw std::runtime_error("data_product object Error: could not find data_product component_object for " + obj->get_value_as_string("object") + " in Registry"); } @@ -520,7 +521,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr storageLocationObj = ApiObject::from_json( j_tmp2 ); if (storageLocationObj->is_empty()){ - APILogger->error("data_product object Error: could not find storage_location for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error("data_product object Error: could not find storage_location for {0} in registry", obj->get_value_as_string("object")); throw std::runtime_error("data_product object Error: could not find storage_location for " + obj->get_value_as_string("object") + " in Registry"); } @@ -529,7 +530,7 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da if (storageRootObj->is_empty()){ std::string obj_str = obj->get_value_as_string("object"); - APILogger->error("data_product object Error: could not find storage_root for {0} in registry", obj_str); + logger::get_logger()->error("data_product object Error: could not find storage_root for {0} in registry", obj_str); throw std::runtime_error("data_product object Error: could not find storage_root for " + obj_str + " in Registry"); } @@ -556,7 +557,7 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentWrite = it->second; if(! file_exists(currentWrite.get_path().string())){ - APILogger->error("File Error: Cannot Find file for write", currentWrite.get_use_data_product()); + logger::get_logger()->error("File Error: Cannot Find file for write", currentWrite.get_use_data_product()); throw std::runtime_error("File Error Cannot Find file for write: " + currentWrite.get_use_data_product()); } @@ -682,7 +683,7 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentOutput = it->second; Json::Value output = currentOutput.get_component_object()->get_uri(); patch_data["outputs"].append(output); - APILogger->info("Writing {} to local registry", currentOutput.get_use_data_product()); + logger::get_logger()->info("Writing {} to local registry", currentOutput.get_use_data_product()); } } @@ -692,12 +693,12 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentInput = it->second; Json::Value input = currentInput.get_component_object()->get_uri(); patch_data["inputs"].append(input); - APILogger->info("Writing {} to local registry", currentInput.get_use_data_product()); + logger::get_logger()->info("Writing {} to local registry", currentInput.get_use_data_product()); } } std::string code_run_endpoint = "code_run/" + std::to_string(code_run_->get_id()); - APILogger->info("Code Run: {0}", code_run_endpoint); + logger::get_logger()->info("Code Run: {0}", code_run_endpoint); Json::Value j_code_run = api_->patch(code_run_endpoint, patch_data, token_); this-> code_run_ = ApiObject::from_json( j_code_run ); diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 745fedc..9185d52 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -13,6 +13,11 @@ static size_t write_file_(char*ptr, size_t size, size_t nmemb, void* userdata ) return written_n_; } +API::sptr API::construct( const std::string& url_root ) +{ + return API::sptr( new API( url_root ) ); +} + std::string url_encode( const std::string& url) { CURL *curl_ = curl_easy_init(); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); @@ -26,14 +31,14 @@ CURL *API::setup_json_session_(std::string &addr_path, std::string *response, if (!token.empty()) { - APILogger->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug("Adding token: {0} to headers", token); struct curl_slist *headers = NULL; headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, headers); } - APILogger->debug("API:JSONSession: Attempting to access: " + addr_path); + logger::get_logger()->debug("API:JSONSession: Attempting to access: " + addr_path); curl_easy_setopt(curl_, CURLOPT_URL, addr_path.c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_str_); @@ -55,7 +60,7 @@ void API::download_file(const ghc::filesystem::path &url, CURL *curl_ = curl_easy_init(); FILE *file_ = fopen(out_path.string().c_str(), "wb"); - APILogger->debug("API: Downloading file '{0}' -> '{1}'", url.string(), + logger::get_logger()->debug("API: Downloading file '{0}' -> '{1}'", url.string(), out_path.string()); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); curl_easy_setopt(curl_, CURLOPT_URL, url.c_str()); @@ -68,7 +73,7 @@ void API::download_file(const ghc::filesystem::path &url, CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, FILE *file) { CURL *curl_ = curl_easy_init(); - APILogger->debug("API:DownloadSession: Attempting to access: " + + logger::get_logger()->debug("API:DownloadSession: Attempting to access: " + addr_path.string()); curl_easy_setopt(curl_, CURLOPT_URL, addr_path.string().c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); @@ -104,7 +109,7 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons JSONCPP_STRING err; if (http_code == 0) { - APILogger->error("API:Request: Request to '{0}' returned no response", + logger::get_logger()->error("API:Request: Request to '{0}' returned no response", search_str_); throw rest_apiquery_error("No response was given"); } @@ -119,7 +124,7 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons if (!json_reader_->parse(response_str_.c_str(), response_str_.c_str() + response_str_len_, &root_, &err)) { - APILogger->error("API:Query: Response string '{0}' is not JSON parsable. " + logger::get_logger()->error("API:Query: Response string '{0}' is not JSON parsable. " "Return Code was {1}", response_str_, http_code); throw rest_apiquery_error( @@ -211,7 +216,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat const std::string url_path_ = url_root_ + API::append_with_forward_slash(addr_path); const std::string data_ = json_to_string(post_data); - APILogger->debug("API:Post: Post Data\n{0}", data_); + logger::get_logger()->debug("API:Post: Post Data\n{0}", data_); long return_code_; std::string response_; CURL *curl_ = curl_easy_init(); @@ -220,7 +225,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat headers = curl_slist_append(headers, "Content-Type: application/json"); if (!token.empty()) { - APILogger->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug("Adding token: {0} to headers", token); headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); } @@ -243,7 +248,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat } else { curl_easy_cleanup(curl_); curl_global_cleanup(); - APILogger->error("API:Post: Post to '{0}' returned no response", url_path_); + logger::get_logger()->error("API:Post: Post to '{0}' returned no response", url_path_); throw rest_apiquery_error("No response was given"); } @@ -255,7 +260,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat } else if (http_code == 409) { - APILogger->info("API:Post Entry Exists attempting to return entry"); + logger::get_logger()->info("API:Post Entry Exists attempting to return entry"); return get_request(API::append_with_forward_slash(addr_path) + json_to_query_string(post_data))[0]; } @@ -278,7 +283,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat if (!json_reader_->parse(response_.c_str(), response_.c_str() + response_str_len_, &root_, &err)) { - APILogger->error("API:Post: Response string '{0}' is not JSON parsable. " + logger::get_logger()->error("API:Post: Response string '{0}' is not JSON parsable. " "Return Code was {1}", response_, return_code_); throw rest_apiquery_error( diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index 6116ac6..4a8ad95 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -2,7 +2,7 @@ namespace FairDataPipeline { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { - APILogger->debug("LocalFileSystem: Reading configuration file '{0}'", + logger::get_logger()->debug("LocalFileSystem: Reading configuration file '{0}'", yaml_path.string().c_str()); return YAML::LoadFile(yaml_path.string().c_str()); } diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 9f7bd98..9e14543 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,20 +1,92 @@ #include "fdp/utilities/logging.hxx" +#include +#include "spdlog/sinks/stdout_color_sinks.h" namespace FairDataPipeline { +namespace logging { - /*! ************************************************************************** - * @brief Create a the global logger pointer - * @author K. Zarebski (UKAEA) - * - * @return a shared pointer to the global logger instance - ****************************************************************************/ - static logger::sptr create_logger() { - if (!spdlog::get("FDPAPI")) { - return spdlog::stdout_color_st("FDPAPI"); - } else { - return spdlog::get("FDPAPI"); + MsgLogger::MsgLogger( enum LOG_LEVEL msg_lvl, Sink& sink ) + : _msg_lvl( msg_lvl ), _sink(sink) + { + } + + MsgLogger Logger::info() + { + MsgLogger msg_logger( INFO, _sink ); + msg_logger << "[INFO ]"; + return msg_logger; + } + + MsgLogger Logger::debug() + { + MsgLogger msg_logger( DEBUG, _sink ); + msg_logger << "[ DEBUG ]"; + return msg_logger; + } + + MsgLogger Logger::trace() + { + MsgLogger msg_logger( TRACE, _sink ); + msg_logger << "[TRACE ]"; + return msg_logger; + } + + MsgLogger Logger::warn() + { + MsgLogger msg_logger( WARN, _sink ); + msg_logger << "[WARN ]"; + return msg_logger; + } + + MsgLogger Logger::critical() + { + MsgLogger msg_logger( WARN, _sink ); + msg_logger << "[CRITICAL]"; + return msg_logger; + } + + MsgLogger Logger::error() + { + MsgLogger msg_logger( ERROR, _sink ); + msg_logger << "[ERROR ]"; + return msg_logger; + } +} + + + logger::sptr logger::_instance = NULL; + + static logger::sptr _create_logger() + { + logger::sptr the_logger = NULL; + + if (!spdlog::get("FDPAPI")) + { + the_logger = spdlog::stdout_color_st("FDPAPI"); + } else + { + the_logger = spdlog::get("FDPAPI"); } + + return the_logger; + } + + logger::sptr logger::get_logger() + { +#if 1 + if( NULL == logger::_instance ) + { + logger::_instance = _create_logger(); + spdlog::set_default_logger(logger::_instance); + } + + return logger::_instance; +#else + + static Logger logger( INFO, std::cout ); + return logger; +#endif + } - const logger::sptr APILogger = create_logger(); } // namespace FairDataPipeline diff --git a/test/test_api.cxx b/test/test_api.cxx index 50ccd38..f31c541 100644 --- a/test/test_api.cxx +++ b/test/test_api.cxx @@ -26,19 +26,20 @@ class ApiTest : public ::testing::Test { } ghc::filesystem::path read_csv_config = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; std::string api_url_ = "http://127.0.0.1:8000/api"; - FairDataPipeline::API *api_ = new FairDataPipeline::API(api_url_); + + FairDataPipeline::API::sptr api_ = FairDataPipeline::API::construct(api_url_); + std::string token = read_token(ghc::filesystem::path(getHomeDirectory()) / ".fair" / "registry" / "token"); void TearDown() override { - delete api_; } }; //! [TestGetById] TEST_F(ApiTest, TestGetById) { Json::Value author = api_->get_request(std::string("author/?name=Interface%20Test"))[0]["url"]; - APILogger->info("Author: {0}", author.asString()); + logger::get_logger()->info("Author: {0}", author.asString()); int author_id_ = ApiObject::get_id_from_string( author.asString() ); ASSERT_EQ(api_->get_by_id(std::string("author"), author_id_)["name"].asString(), std::string("Interface Test")); } //! [TestGetById] diff --git a/test/test_config.cxx b/test/test_config.cxx index a38335a..6da846b 100644 --- a/test/test_config.cxx +++ b/test/test_config.cxx @@ -19,7 +19,8 @@ using namespace FairDataPipeline; class ConfigTest : public ::testing::Test { protected: void SetUp() override {} - std::string getHomeDirectory() { + + static std::string getHomeDirectory() { std::string HomeDirectory; #ifdef _WIN32 HomeDirectory = getenv("HOMEDRIVE"); @@ -30,35 +31,40 @@ class ConfigTest : public ::testing::Test { return HomeDirectory; } - Config::sptr config(bool use_local = true, std::string config = "write_csv.yaml") { + static Config::sptr config(bool use_local = true, std::string config = "write_csv.yaml") { + + std::string home_dir = getHomeDirectory(); + std::string token = + read_token (ghc::filesystem::path( home_dir) / ".fair" / "registry" / "token"); - const ghc::filesystem::path config_path_ = - ghc::filesystem::path(TESTDIR) / "data" / config; - const ghc::filesystem::path script_path_ = - ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); + const ghc::filesystem::path config_path_ = + ghc::filesystem::path(TESTDIR) / "data" / config; + const ghc::filesystem::path script_path_ = + ghc::filesystem::path(TESTDIR) / "test_script.sh"; + logger::get_logger()->set_level(spdlog::level::debug); - return Config::construct(config_path_, script_path_, token, - static_cast(use_local)); + return Config::construct(config_path_, script_path_, token, + use_local ? RESTAPI::LOCAL : RESTAPI::REMOTE ); } - std::string token = - read_token(ghc::filesystem::path(getHomeDirectory()) / - ".fair" / "registry" / "token"); void TearDown() override {} }; TEST_F(ConfigTest, TestMetaData) { - EXPECT_EQ(config()->meta_data_()["description"].as(), "Write csv file"); - EXPECT_EQ(config()->meta_data_()["local_data_registry_url"].as(), "http://127.0.0.1:8000/api/"); - EXPECT_EQ(config()->meta_data_()["remote_data_registry_url"].as(), "https://data.scrc.uk/api/"); - EXPECT_EQ(config()->meta_data_()["default_input_namespace"].as(), "testing"); - EXPECT_EQ(config()->meta_data_()["default_output_namespace"].as(), "testing"); - //EXPECT_EQ(config()->meta_data_()["write_data_store"].as(), "tmp"); - EXPECT_EQ(config()->meta_data_()["local_repo"].as(), "./"); - EXPECT_EQ(config()->meta_data_()["public"].as(), "true"); - EXPECT_EQ(config()->meta_data_()["latest_commit"].as(), "52008720d240693150e96021ea34ac6fffe05870"); - EXPECT_EQ(config()->meta_data_()["remote_repo"].as(), "https://github.com/FAIRDataPipeline/cppDataPipeline"); + + auto cnf = config(); + auto meta = cnf->meta_data_(); + + EXPECT_EQ( meta["description"].as(), "Write csv file"); + EXPECT_EQ( meta["local_data_registry_url"].as(), "http://127.0.0.1:8000/api/"); + EXPECT_EQ( meta["remote_data_registry_url"].as(), "https://data.scrc.uk/api/"); + EXPECT_EQ( meta["default_input_namespace"].as(), "testing"); + EXPECT_EQ( meta["default_output_namespace"].as(), "testing"); + //EXPECT_EQ(cnf->meta_data_()["write_data_store"].as(), "tmp"); + EXPECT_EQ( meta["local_repo"].as(), "./"); + EXPECT_EQ( meta["public"].as(), "true"); + EXPECT_EQ( meta["latest_commit"].as(), "52008720d240693150e96021ea34ac6fffe05870"); + EXPECT_EQ( meta["remote_repo"].as(), "https://github.com/FAIRDataPipeline/cppDataPipeline"); } TEST_F(ConfigTest, TestLinkWrite){ diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index 7bdd5bf..e5f8b93 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -39,11 +39,11 @@ TEST_F(PimplTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "data" / "write_csv.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); - DataPipeline dp = DataPipeline(config_path_.string(), script_path_.string(), token, spdlog::level::debug); + logger::get_logger()->set_level(spdlog::level::debug); + DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); std::string data_product = "test/csv"; - ghc::filesystem::path currentLink = ghc::filesystem::path(dp.link_write(data_product)); + ghc::filesystem::path currentLink = ghc::filesystem::path(dp->link_write(data_product)); EXPECT_GT(currentLink.string().size(), 1); std::ofstream testCSV; @@ -51,16 +51,16 @@ TEST_F(PimplTest, TestDataPipelineInit) { testCSV << "Test"; testCSV.close(); - dp.finalise(); + dp->finalise(); ghc::filesystem::path config_path_read_ = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; - dp = DataPipeline(config_path_read_.string(), script_path_.string(), token, spdlog::level::debug); + dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); - currentLink = dp.link_read(data_product); + currentLink = dp->link_read(data_product); EXPECT_GT(currentLink.string().size(), 1); - dp.finalise(); + dp->finalise(); } diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 9b4caac..3f5543a 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -25,19 +25,19 @@ class RegistryTest : public ::testing::Test { return HomeDirectory; } - DataPipeline *init_pipeline(bool use_local = true) { + DataPipeline::sptr init_pipeline(bool use_local = true) { const ghc::filesystem::path config_path_ = ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); + logger::get_logger()->set_level(spdlog::level::debug); - return new DataPipeline( + return DataPipeline::construct( config_path_.string(), script_path_.string(), token, - spdlog::level::debug ); + logger::LOG_LEVEL::log_level_debug ); } std::string token = @@ -51,8 +51,9 @@ TEST_F(RegistryTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); - DataPipeline(config_path_.string(), script_path_.string(), token, spdlog::level::debug); + logger::get_logger()->set_level(spdlog::level::debug); + + auto dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); } TEST_F(RegistryTest, TestLogLevelSet) { @@ -72,3 +73,12 @@ TEST_F(RegistryTest, TestHashFile) { calculate_hash_from_file(ghc::filesystem::path(TESTDIR) / "config.yaml"); std::cout << "HASH: " << file_hash_ << std::endl; } + +TEST_F(RegistryTest, TestLogger) { + logging::OStreamSink sink( logging::INFO, std::cout ); + logging::Logger logger( logging::INFO, sink ); + logger.trace() << 1 << " " << "TRACE" << "\n"; + logger.info() << 1 << " " << "INFO" << "\n"; + logger.warn() << 1 << " " << "WARN" << "\n"; + //ASSERT_EQ(0,1) ; +} From 64fc506048cd105145b04bd08c1114be402027c5 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Thu, 31 Mar 2022 23:24:51 +0100 Subject: [PATCH 08/16] Replace spd logger complete --- include/fdp/fdp.hxx | 4 +- include/fdp/registry/data_io.hxx | 5 +- include/fdp/utilities/logging.hxx | 187 ++++++++++++++---------- src/fdp.cxx | 33 +++-- src/objects/config.cxx | 169 +++++++++++++++------- src/registry/api.cxx | 57 +++++--- src/registry/data_io.cxx | 6 +- src/utilities/logging.cxx | 228 +++++++++++++++++++++--------- test/test_api.cxx | 4 +- test/test_config.cxx | 2 +- test/test_pimpl.cxx | 8 +- test/test_registry.cxx | 26 ++-- 12 files changed, 490 insertions(+), 239 deletions(-) diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 03a7269..00589f6 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -27,7 +27,7 @@ namespace FairDataPipeline { const std::string &config_file_path, const std::string &script_file_path, std::string token = "", - logger::LOG_LEVEL log_level = logger::LOG_LEVEL::log_level_info ); + enum logging::LOG_LEVEL log_level = logging::LOG_LEVEL::INFO ); /** @@ -71,7 +71,7 @@ namespace FairDataPipeline { const std::string &config_file_path, const std::string &script_file_path, std::string token = "", - logger::LOG_LEVEL log_level = logger::LOG_LEVEL::log_level_info); + enum logging::LOG_LEVEL log_level = logging::LOG_LEVEL::INFO); DataPipeline(const DataPipeline &rhs) = delete; diff --git a/include/fdp/registry/data_io.hxx b/include/fdp/registry/data_io.hxx index 884409d..60f47c0 100644 --- a/include/fdp/registry/data_io.hxx +++ b/include/fdp/registry/data_io.hxx @@ -88,8 +88,9 @@ ghc::filesystem::path create_estimate(T &value, toml_out_.close(); - logger::sptr the_logger = logger::get_logger(); - the_logger->debug( "FileSystem:CreateEstimate: Wrote point estimate to '{0}'", output_filename_.string()); + auto the_logger = logger::get_logger(); + the_logger->debug() + << "FileSystem:CreateEstimate: Wrote point estimate to '" << output_filename_.string() << "'"; return output_filename_; } diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index bee2871..a610465 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -30,6 +30,8 @@ namespace FairDataPipeline { namespace logging { + std::string curtime(); + enum LOG_LEVEL { TRACE = 0, @@ -41,147 +43,190 @@ namespace FairDataPipeline { OFF = 6 }; + const std::string& _LOG_LEVEL_2_str( enum LOG_LEVEL lvl ); + struct ISink { - virtual int log( enum LOG_LEVEL msg_lvl, const std::string& s) = 0; - virtual ~ISink(){}; + typedef std::shared_ptr< ISink > sptr; + virtual int log( enum LOG_LEVEL msg_lvl, const std::string& msg ) = 0; + virtual ~ISink(){}; }; + class Logger; + + class SinkFormatter; + typedef std::shared_ptr< SinkFormatter > SinkFormatterPtr; + + class Sink : public ISink { public: - Sink( enum LOG_LEVEL log_lvl ) : ISink(), _log_lvl(log_lvl) {} + typedef std::shared_ptr< Sink > sptr; - virtual int log( enum LOG_LEVEL msg_lvl, const std::string& s){return 1;} + std::string apply_formatter( Logger* loggher, enum LOG_LEVEL msg_level, const std::string& msg ); - template< typename T > - int execute( enum LOG_LEVEL msg_lvl, const T& s) - { - if( this->should_log( msg_lvl ) ) - { - std::ostringstream oss; - oss << s; + int execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ); - this->log( msg_lvl, oss.str() ); - } - return 0; - } + enum LOG_LEVEL log_level(); + void log_level( enum LOG_LEVEL log_lvl ); - enum LOG_LEVEL log_level(){ return _log_lvl;} - void log_level( enum LOG_LEVEL log_lvl ){ _log_lvl = log_lvl;} + bool should_log( enum LOG_LEVEL msg_lvl ); - bool should_log( enum LOG_LEVEL msg_lvl ){ return msg_lvl >= this->log_level();} + void set_formatter( SinkFormatterPtr fmtr ){_fmtr = fmtr;} + + protected: + Sink( enum LOG_LEVEL log_lvl ); + //std::string do_header( enum LOG_LEVEL msg_lvl, Logger* logger ); private: enum LOG_LEVEL _log_lvl; + SinkFormatterPtr _fmtr; }; class OStreamSink : public Sink { public: - OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os){ ;} + typedef std::shared_ptr< OStreamSink > sptr; + + static sptr create( enum LOG_LEVEL lvl, std::ostream& os ) + { + return sptr( new OStreamSink( lvl, os ) ); + } + int log( enum LOG_LEVEL msg_lvl, const std::string& s) { - if( this->should_log( msg_lvl ) ) - _os << s; + //if( this->should_log( msg_lvl ) ) + _os << s; return 0; } private: + + OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os){ ;} + std::ostream& _os; }; class CompositeSink : public Sink { public: - CompositeSink(enum LOG_LEVEL log_lvl) : Sink(log_lvl){ ;} + typedef std::shared_ptr< CompositeSink > sptr; - int log( enum LOG_LEVEL msg_lvl, const std::string& s) - { - if( this->should_log( msg_lvl ) ) - { - for( int i = 0; i < _sinks.size(); ++i ) - { - Sink* sink = _sinks[i]; - sink->log( msg_lvl, s ); - } - } - return 0; - } + static sptr create( enum LOG_LEVEL lvl ){ return sptr( new CompositeSink( lvl ) ); } + - void add_sink( Sink* sink ){ _sinks.push_back( sink ); } + int log( enum LOG_LEVEL msg_lvl, const std::string& s); + void add_sink( Sink::sptr sink ){ _sinks.push_back( sink ); } private: - std::vector< Sink* > _sinks; - }; + std::vector< Sink::sptr > _sinks; + CompositeSink(enum LOG_LEVEL log_lvl) : Sink(log_lvl){ ;} + }; + struct ISinkFormatter + { + virtual std::string header(enum LOG_LEVEL msg_lvl, Logger* logger )= 0; + virtual ~ISinkFormatter(){;} + }; - class MsgLogger + class SinkFormatter : public ISinkFormatter { public: - MsgLogger( enum LOG_LEVEL msg_lvl, Sink& sink ); + typedef std::shared_ptr< SinkFormatter > sptr; + + static sptr create(){ return sptr( new SinkFormatter() ); } + + ~SinkFormatter() + { + } - template< typename T > - MsgLogger& operator<<( const T& s) - { - this->_sink.execute( _msg_lvl, s ); - return *this; - } + std::string header( enum LOG_LEVEL msg_lvl, Logger* logger ); private: - enum LOG_LEVEL _msg_lvl; - Sink& _sink; + SinkFormatter(){} }; class Logger { public: + typedef std::shared_ptr< Logger > sptr; + class MsgBuilder + { + public: + friend class Logger; + typedef std::shared_ptr< MsgBuilder > sptr; + + ~MsgBuilder(); + + template< typename T > + MsgBuilder& operator<<( const T& s) + { + if( _logger->sink()->should_log( _msg_lvl ) ) + { + _oss << s; + } + return *this; + } + + private: + MsgBuilder( enum LOG_LEVEL msg_lvl, Logger* logger ); + MsgBuilder( const MsgBuilder& rhs ); + + Logger* _logger; + enum LOG_LEVEL _msg_lvl; + + std::ostringstream _oss; + }; + + static sptr create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name="" ) + { + return sptr( new Logger(lvl, sink, name)); + } + - Logger( enum LOG_LEVEL lvl, Sink& sink ) - : _sink(sink) + MsgBuilder info(); + MsgBuilder debug(); + MsgBuilder trace(); + MsgBuilder warn(); + MsgBuilder critical(); + MsgBuilder error(); + + Sink::sptr sink(){ return _sink; } + const std::string& name() const { return _name;} + + void set_level( enum LOG_LEVEL lvl){ _log_lvl = lvl;} + enum LOG_LEVEL get_level() const { return _log_lvl;} + + protected: + Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) + : _sink(sink), _name(name) { _log_lvl = lvl; } - MsgLogger info(); - MsgLogger debug(); - MsgLogger trace(); - MsgLogger warn(); - MsgLogger critical(); - MsgLogger error(); private: enum LOG_LEVEL _log_lvl; - Sink& _sink; + Sink::sptr _sink; + std::string _name; }; } class logger { public: - enum LOG_LEVEL - { - log_level_trace = 0, - log_level_debug = 1, - log_level_info = 2, - log_level_warn = 3, - log_level_error = 4, - log_level_critical = 5, - log_level_off = 6 - }; - - typedef std::shared_ptr sptr; - - static sptr get_logger(); + typedef std::shared_ptr< logging::Logger > logger_sptr; + + static logger_sptr get_logger(); private: - static sptr _instance; + static logger_sptr _instance; }; } // namespace FairDataPipeline diff --git a/src/fdp.cxx b/src/fdp.cxx index ad4c20f..95ee358 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -14,6 +14,7 @@ #include "fdp/utilities/logging.hxx" namespace FairDataPipeline { +#if 0 spdlog::level::level_enum LOG_LEVEL_2_spdlog( logger::LOG_LEVEL fdp_log_level ) { std::map< logger::LOG_LEVEL, spdlog::level::level_enum > m; @@ -28,6 +29,7 @@ namespace FairDataPipeline { return m[ fdp_log_level ]; } +#endif /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods @@ -54,7 +56,7 @@ namespace FairDataPipeline { impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - logger::LOG_LEVEL log_lvl, + enum logging::LOG_LEVEL log_lvl, RESTAPI api_location = RESTAPI::LOCAL); impl(const impl &dp) = delete; @@ -74,7 +76,7 @@ namespace FairDataPipeline { static sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - logger::LOG_LEVEL log_level, + enum logging::LOG_LEVEL log_level, RESTAPI api_location = RESTAPI::LOCAL); @@ -124,7 +126,7 @@ namespace FairDataPipeline { DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - logger::LOG_LEVEL log_level, + enum logging::LOG_LEVEL log_level, RESTAPI api_location) { @@ -139,10 +141,10 @@ DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::pa DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - logger::LOG_LEVEL log_level, + enum logging::LOG_LEVEL log_level, RESTAPI api_location) { - spdlog::level::level_enum spd_log_lvl = LOG_LEVEL_2_spdlog( log_level ); + //spdlog::level::level_enum spd_log_lvl = LOG_LEVEL_2_spdlog( log_level ); this->config_ = Config::construct(config_file_path, script_file_path, token, api_location); @@ -159,14 +161,14 @@ DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, token); #else - logger::sptr the_logger = logger::get_logger(); + auto the_logger = logger::get_logger(); //spdlog::set_default_logger(the_logger); - the_logger->set_level( spd_log_lvl ); - the_logger->info("\n[Configuration]\n\t- Config Path: {0}\n\t- API Root: " - "{1}\n\t- FDP API Token: {2}", - config_file_path.string(), api_root_, - token); + //the_logger->set_level( log_level ); + the_logger->info() << "\n[Configuration]\n\t- Config Path:" << config_file_path.string() + << "\n\t- API Root: " + << api_root_ + << "\n\t- FDP API Token: " << token; #endif @@ -192,7 +194,7 @@ DataPipeline::sptr DataPipeline::construct( const std::string &config_file_path, const std::string &script_file_path, std::string token, - logger::LOG_LEVEL log_level ) + enum logging::LOG_LEVEL log_level ) { return DataPipeline::sptr( new DataPipeline( config_file_path, @@ -206,12 +208,13 @@ DataPipeline::DataPipeline( const std::string &config_file_path, const std::string &script_file_path, std::string token, - logger::LOG_LEVEL log_level ) + logging::LOG_LEVEL log_level ) : pimpl_( DataPipeline::impl::construct(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token, log_level)) { - logger::sptr logger = logger::get_logger(); - logger->debug("DataPipeline: Initialising session '{0}'", pimpl_->get_code_run_uuid()); + // logger::spd_sptr logger = logger::get_spd_logger(); + logger::get_logger()->debug() << "DataPipeline: Initialising session '" + << pimpl_->get_code_run_uuid() << "'"; } ghc::filesystem::path FairDataPipeline::DataPipeline::link_read(std::string &data_product){ diff --git a/src/objects/config.cxx b/src/objects/config.cxx index 2722b60..7f64839 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -32,8 +32,8 @@ Config::~Config() { } YAML::Node FairDataPipeline::Config::parse_yaml(ghc::filesystem::path yaml_path) { - logger::get_logger()->debug("[Config]: Reading configuration file '{0}'", - yaml_path.string().c_str()); + logger::get_logger()->debug() + << "[Config]: Reading configuration file '" << yaml_path.string().c_str() << "'"; return YAML::LoadFile(yaml_path.string().c_str()); } @@ -96,64 +96,76 @@ void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, RESTAPI api_location) { config_data_ = parse_yaml(yaml_path); if (!config_data_["run_metadata"]) { - logger::get_logger()->error( - "Failed to obtain run metadata, the key 'run_metadata' is not" - " present in the configuration"); + logger::get_logger()->error() + << "Failed to obtain run metadata, the key 'run_metadata' is not" + " present in the configuration"; throw config_parsing_error("No run metadata provided"); } if (api_location == RESTAPI::LOCAL && !meta_data_()["local_data_registry_url"]) { - logger::get_logger()->error("Could not determine URL for local repository, key " - "'local_data_registry_url' " - "is absent from configuration"); + logger::get_logger()->error() + << "Could not determine URL for local repository, key " + "'local_data_registry_url' " + "is absent from configuration"; throw config_parsing_error("Failed to read local repository url"); } else if (api_location == RESTAPI::REMOTE && !meta_data_()["remote_data_registry_url"]) { - logger::get_logger()->error("Could not determine URL for remote repository, key " - "'remote_data_registry_url' " - "is absent from configuration"); + logger::get_logger()->error() + <<"Could not determine URL for remote repository, key " + "'remote_data_registry_url' " + "is absent from configuration"; throw config_parsing_error("Failed to read remote repository url"); } else if (!(api_location == RESTAPI::LOCAL || api_location == RESTAPI::REMOTE)) { - logger::get_logger()->error("Unrecognised API location"); + logger::get_logger()->error() << "Unrecognised API location"; throw config_parsing_error("Failed to resolve registry location"); } if (!meta_data_()["write_data_store"]) { - logger::get_logger()->error("Failed to Read: [\"write_data_store\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"write_data_store\"] from " + << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"write_data_store\"] from " + yaml_path.string()); } if (!meta_data_()["default_input_namespace"]) { - logger::get_logger()->error("Failed to Read: [\"default_input_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"default_input_namespace\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"default_input_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["default_output_namespace"]) { - logger::get_logger()->error("Failed to Read: [\"default_output_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"default_output_namespace\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"default_output_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["latest_commit"]) { - logger::get_logger()->error("Failed to Read: [\"latest_commit\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"latest_commit\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"latest_commit\"] from " + yaml_path.string()); } if (!meta_data_()["remote_repo"]) { - logger::get_logger()->error("Failed to Read: [\"remote_repo\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"remote_repo\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"remote_repo\"] from " + yaml_path.string()); } if (!meta_data_()["description"]) { - logger::get_logger()->error("Failed to Read: [\"description\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"description\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"description\"] from " + yaml_path.string()); } if (!file_exists(script_file_path_.string())) { - logger::get_logger()->error("Submission script: {0} does not exist", script_file_path_.string()); + logger::get_logger()->error() + << "Submission script: " + << script_file_path_.string() + << " does not exist"; throw std::runtime_error("Submission script: " + script_file_path_.string() + " does not exist"); } @@ -184,8 +196,9 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { meta_data_()["local_data_registry_url"].as()); } - logger::get_logger()->info("Reading {0} from local filestore", - config_file_path_.string()); + logger::get_logger()->info() << "Reading " + << config_file_path_.string() + << " from local filestore"; // Create and API object as a shared pointer api_ = API::construct(api_url_); @@ -197,7 +210,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->user_ = ApiObject::from_json( j [0]); if (user_->is_empty()) { - logger::get_logger()->error("User: Admin Not Found"); + logger::get_logger()->error() << "User: Admin Not Found"; throw std::runtime_error("User: Admin Not Found"); } @@ -211,8 +224,8 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->author_ = ApiObject::from_json( j_author ); if (author_->is_empty()) { - logger::get_logger()->error( - "Author for User Admin not found please ensure you have run fair init"); + logger::get_logger()->error() + << "Author for User Admin not found please ensure you have run fair init"; throw std::runtime_error( "Author Not Found: Please ensure you have run fair init"); } @@ -258,7 +271,10 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { config_value_["file_type"] = config_file_type_url_; - logger::get_logger()->info("Writing config file {0} to registry", config_file_path_.string()); + logger::get_logger()->info() + << "Writing config file " + << config_file_path_.string() + << " to registry", config_file_path_.string(); Json::Value j_config_obj = api_->post("object", config_value_, token_); @@ -296,7 +312,10 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { script_value_["filetype"] = script_file_type_url_.asString(); script_value_["storage_location"] = script_storage_location_->get_uri(); - logger::get_logger()->info("Writing script file {0} to registry", script_file_path_.string()); + logger::get_logger()->info() + << "Writing script file " + << script_file_path_.string() + << " to registry"; Json::Value j_script_obj = api_->post("object", script_value_, token_); this->script_obj_ = ApiObject::from_json( j_script_obj ); @@ -336,12 +355,15 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { code_run_value_["input_urls"] = Json::arrayValue; code_run_value_["output_urls"] = Json::arrayValue; - logger::get_logger()->info("Writing new code run to registry"); + logger::get_logger()->info() << "Writing new code run to registry"; Json::Value j_code_run = api_->post("code_run", code_run_value_, token_); this->code_run_ = ApiObject::from_json( j_code_run ); - logger::get_logger()->info("Code run {0} successfully generated", code_run_->get_value_as_string("uuid")); + logger::get_logger()->info() + << "Code run " + << code_run_->get_value_as_string("uuid") + << " successfully generated"; } std::string Config::get_config_directory() const{ @@ -355,11 +377,13 @@ std::string Config::get_code_run_uuid() const{ ghc::filesystem::path Config::link_write( const std::string& data_product){ if (!config_has_writes()){ - logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } - //logger::get_logger()->info("Node Type: {0}", config_writes_().Type()); + //logger::get_logger()->info() << "Node Type: " + //<< config_writes_().Type()); YAML::Node currentWrite; @@ -375,30 +399,44 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ } } else{ - logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentWrite) { - logger::get_logger()->error("Config Error: Cannot Find {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find " + << data_product + << " in writes"; + throw config_parsing_error("Config Error: cannot find " + data_product + "in writes"); } if(!currentWrite["description"]) { - logger::get_logger()->error("Config Error: Cannot Find description of {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find description of " + << data_product + << " in writes"; throw config_parsing_error("Config Error: cannot find description of " + data_product + "in writes"); } if(!currentWrite["file_type"]) { - logger::get_logger()->error("Config Error: Cannot Find file_type of {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find file_type of " + << data_product + << " in writes"; throw config_parsing_error("Config Error: cannot find file_type of " + data_product + "in writes"); } if(!currentWrite["use"]["version"]){ - logger::get_logger()->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info() + << "Use: Version not found in " + << data_product + << ", using version 0.0.1 by default"; currentWrite["use"]["version"] = "0.0.1"; } @@ -413,7 +451,7 @@ ghc::filesystem::path Config::link_write( const std::string& data_product){ std::string filename_("dat-" + generate_random_hash() + "." + currentWrite["file_type"].as()); ghc::filesystem::path path_ = ghc::filesystem::path(meta_data_()["write_data_store"].as()) / currentWrite["use"]["namespace"].as() / currentWrite["use"]["data_product"].as() / filename_; - logger::get_logger()->info("Link Path: {0}", path_.string()); + logger::get_logger()->info() << "Link Path: " << path_.string(); // Create Directory ghc::filesystem::create_directories(path_.parent_path().string()); @@ -450,18 +488,25 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da } } else{ - logger::get_logger()->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentRead) { - logger::get_logger()->error("Config Error: Cannot Find {0} in reads", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find " + << data_product + << " in reads"; throw config_parsing_error("Config Error: cannot find " + data_product + "in reads"); } if(!currentRead["use"]["version"]){ - logger::get_logger()->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info() + << "Use: Version not found in " + << data_product + << ", using version 0.0.1 by default"; currentRead["use"]["version"] = "0.0.1"; } @@ -480,7 +525,10 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr namespaceObj = ApiObject::from_json( j_namespace ); if (namespaceObj->is_empty()){ - logger::get_logger()->error("Namespace Error: could not find namespace {0} in registry", currentRead["use"]["namespace"].as()); + logger::get_logger()->error() + << "Namespace Error: could not find namespace " + << currentRead["use"]["namespace"].as() + << " in registry"; throw std::runtime_error("Namespace Error: could not find namespace " + currentRead["use"]["namespace"].as() + " in Registry"); } @@ -493,7 +541,10 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr dataProductObj = ApiObject::from_json( j_data_prod_obj ); //ApiObject dataProductObj = ApiObject(api_->get_by_json_query("data_product", dataProductData)[0]); if (dataProductObj->is_empty()){ - logger::get_logger()->error("data_product Error: could not find data_product {0} in registry", currentRead["use"]["data_product"].as()); + logger::get_logger()->error() + << "data_product Error: could not find data_product " + << currentRead["use"]["data_product"].as() + << " in registry"; throw std::runtime_error("Namespace Error: could not find data_product " + currentRead["use"]["data_product"].as() + " in Registry"); } @@ -501,7 +552,10 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr obj = ApiObject::from_json( _j_ ); if (obj->is_empty()){ - logger::get_logger()->error("data_product Error: could not find data_product object {0} in registry", dataProductObj->get_value_as_string("object")); + logger::get_logger()->error() + << "data_product Error: could not find data_product object " + << dataProductObj->get_value_as_string("object") + << " in registry"; throw std::runtime_error("Namespace Error: could not find data_product object " + dataProductObj->get_value_as_string("object") + " in Registry"); } @@ -511,7 +565,10 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr componentObj = ApiObject::from_json( j_obj_component ); if (componentObj->is_empty()){ - logger::get_logger()->error("data_product object Error: could not find data_product component_object for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error() + << "data_product object Error: could not find data_product component_object for " + << obj->get_value_as_string("object") + << " in registry"; throw std::runtime_error("data_product object Error: could not find data_product component_object for " + obj->get_value_as_string("object") + " in Registry"); } @@ -521,7 +578,9 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da ApiObject::sptr storageLocationObj = ApiObject::from_json( j_tmp2 ); if (storageLocationObj->is_empty()){ - logger::get_logger()->error("data_product object Error: could not find storage_location for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error() << "data_product object Error: could not find storage_location for " + << obj->get_value_as_string("object") << " in registry"; + throw std::runtime_error("data_product object Error: could not find storage_location for " + obj->get_value_as_string("object") + " in Registry"); } @@ -530,7 +589,10 @@ ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &da if (storageRootObj->is_empty()){ std::string obj_str = obj->get_value_as_string("object"); - logger::get_logger()->error("data_product object Error: could not find storage_root for {0} in registry", obj_str); + + logger::get_logger()->error() + << "data_product object Error: could not find storage_root for " << obj_str << " in registry"; + throw std::runtime_error("data_product object Error: could not find storage_root for " + obj_str + " in Registry"); } @@ -557,7 +619,9 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentWrite = it->second; if(! file_exists(currentWrite.get_path().string())){ - logger::get_logger()->error("File Error: Cannot Find file for write", currentWrite.get_use_data_product()); + logger::get_logger()->error() + << "File Error: Cannot Find file for write" << currentWrite.get_use_data_product(); + throw std::runtime_error("File Error Cannot Find file for write: " + currentWrite.get_use_data_product()); } @@ -683,7 +747,13 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentOutput = it->second; Json::Value output = currentOutput.get_component_object()->get_uri(); patch_data["outputs"].append(output); - logger::get_logger()->info("Writing {} to local registry", currentOutput.get_use_data_product()); + //logger::get_logger()->info() + //<< "Writing + //<< currentOutput.get_use_data_product() + //<< " to local registry"; + logger::get_logger()->info() + << "Writing " << currentOutput.get_use_data_product() << " to local registry"; + } } @@ -693,12 +763,13 @@ void FairDataPipeline::Config::finalise(){ IOObject& currentInput = it->second; Json::Value input = currentInput.get_component_object()->get_uri(); patch_data["inputs"].append(input); - logger::get_logger()->info("Writing {} to local registry", currentInput.get_use_data_product()); + logger::get_logger()->info() + << "Writing " << currentInput.get_use_data_product() << " to local registry"; } } std::string code_run_endpoint = "code_run/" + std::to_string(code_run_->get_id()); - logger::get_logger()->info("Code Run: {0}", code_run_endpoint); + logger::get_logger()->info() << "Code Run: " << code_run_endpoint; Json::Value j_code_run = api_->patch(code_run_endpoint, patch_data, token_); this-> code_run_ = ApiObject::from_json( j_code_run ); diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 9185d52..b8e6eb1 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -31,14 +31,18 @@ CURL *API::setup_json_session_(std::string &addr_path, std::string *response, if (!token.empty()) { - logger::get_logger()->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug() + << "Adding token: " + << token + << " to headers"; struct curl_slist *headers = NULL; headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, headers); } - logger::get_logger()->debug("API:JSONSession: Attempting to access: " + addr_path); + logger::get_logger()->debug() + << "API:JSONSession: Attempting to access: " << addr_path; curl_easy_setopt(curl_, CURLOPT_URL, addr_path.c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_str_); @@ -60,8 +64,10 @@ void API::download_file(const ghc::filesystem::path &url, CURL *curl_ = curl_easy_init(); FILE *file_ = fopen(out_path.string().c_str(), "wb"); - logger::get_logger()->debug("API: Downloading file '{0}' -> '{1}'", url.string(), - out_path.string()); + logger::get_logger()->debug() + << "API: Downloading file '" + << url.string() + << "' -> '" << out_path.string() << "'", url.string(); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); curl_easy_setopt(curl_, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_file_); @@ -73,8 +79,10 @@ void API::download_file(const ghc::filesystem::path &url, CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, FILE *file) { CURL *curl_ = curl_easy_init(); - logger::get_logger()->debug("API:DownloadSession: Attempting to access: " + - addr_path.string()); + + logger::get_logger()->debug() + << "API:DownloadSession: Attempting to access: " << addr_path.string(); + curl_easy_setopt(curl_, CURLOPT_URL, addr_path.string().c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_file_); @@ -109,8 +117,10 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons JSONCPP_STRING err; if (http_code == 0) { - logger::get_logger()->error("API:Request: Request to '{0}' returned no response", - search_str_); + logger::get_logger()->error() + << "API:Request: Request to '" + << search_str_ + << "' returned no response"; throw rest_apiquery_error("No response was given"); } @@ -124,9 +134,11 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons if (!json_reader_->parse(response_str_.c_str(), response_str_.c_str() + response_str_len_, &root_, &err)) { - logger::get_logger()->error("API:Query: Response string '{0}' is not JSON parsable. " - "Return Code was {1}", - response_str_, http_code); + logger::get_logger()->error() + << "API:Query: Response string '" + << response_str_ + << "' is not JSON parsable. Return Code was " + << http_code; throw rest_apiquery_error( "Failed to retrieve information from JSON response string"); } @@ -216,7 +228,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat const std::string url_path_ = url_root_ + API::append_with_forward_slash(addr_path); const std::string data_ = json_to_string(post_data); - logger::get_logger()->debug("API:Post: Post Data\n{0}", data_); + logger::get_logger()->debug() << "API:Post: Post Data\n" << data_; long return_code_; std::string response_; CURL *curl_ = curl_easy_init(); @@ -225,7 +237,10 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat headers = curl_slist_append(headers, "Content-Type: application/json"); if (!token.empty()) { - logger::get_logger()->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug() + << "Adding token: " + << token + << " to headers"; headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); } @@ -248,7 +263,10 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat } else { curl_easy_cleanup(curl_); curl_global_cleanup(); - logger::get_logger()->error("API:Post: Post to '{0}' returned no response", url_path_); + logger::get_logger()->error() + << "API:Post: Post to '" + <info("API:Post Entry Exists attempting to return entry"); + logger::get_logger()->info() + << "API:Post Entry Exists attempting to return entry"; return get_request(API::append_with_forward_slash(addr_path) + json_to_query_string(post_data))[0]; } @@ -283,9 +302,11 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat if (!json_reader_->parse(response_.c_str(), response_.c_str() + response_str_len_, &root_, &err)) { - logger::get_logger()->error("API:Post: Response string '{0}' is not JSON parsable. " - "Return Code was {1}", - response_, return_code_); + logger::get_logger()->error() + << "API:Post: Response string '" + << response_ + << "' is not JSON parsable. Return Code was " + << return_code_; throw rest_apiquery_error( "Failed to retrieve information from JSON response string"); } diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index 4a8ad95..0382a42 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -2,8 +2,10 @@ namespace FairDataPipeline { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { - logger::get_logger()->debug("LocalFileSystem: Reading configuration file '{0}'", - yaml_path.string().c_str()); + logger::get_logger()->debug() + << "LocalFileSystem: Reading configuration file '" + << yaml_path.string() + << "'"; return YAML::LoadFile(yaml_path.string().c_str()); } diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 9e14543..8c715cd 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,92 +1,190 @@ #include "fdp/utilities/logging.hxx" #include +#include +#include +//#include +//#include #include "spdlog/sinks/stdout_color_sinks.h" namespace FairDataPipeline { -namespace logging { + namespace logging { + const std::string& _LOG_LEVEL_2_str( enum LOG_LEVEL lvl ) + { + static const std::string strs[] = {"TRACE","DEBUG","INFO","WARN","ERROR","CRITICAL","OFF"}; + return strs[ lvl ]; + } - MsgLogger::MsgLogger( enum LOG_LEVEL msg_lvl, Sink& sink ) - : _msg_lvl( msg_lvl ), _sink(sink) - { - } + std::string curtime() + { + timeval curTime; + gettimeofday( &curTime, NULL ); + int milli = curTime.tv_usec / 1000; + char buffer[ 80 ]; + strftime( buffer, 80, "%Y-%m-%d %H:%M:%S", localtime( &curTime.tv_sec )); + std::string currentTime( 84, 0 ); - MsgLogger Logger::info() - { - MsgLogger msg_logger( INFO, _sink ); - msg_logger << "[INFO ]"; - return msg_logger; - } - - MsgLogger Logger::debug() - { - MsgLogger msg_logger( DEBUG, _sink ); - msg_logger << "[ DEBUG ]"; - return msg_logger; - } - - MsgLogger Logger::trace() - { - MsgLogger msg_logger( TRACE, _sink ); - msg_logger << "[TRACE ]"; - return msg_logger; - } - - MsgLogger Logger::warn() - { - MsgLogger msg_logger( WARN, _sink ); - msg_logger << "[WARN ]"; - return msg_logger; - } - - MsgLogger Logger::critical() - { - MsgLogger msg_logger( WARN, _sink ); - msg_logger << "[CRITICAL]"; - return msg_logger; - } - - MsgLogger Logger::error() - { - MsgLogger msg_logger( ERROR, _sink ); - msg_logger << "[ERROR ]"; - return msg_logger; - } -} + sprintf( ¤tTime[0], "%s.%03d", buffer, milli ); + return std::move( currentTime ); + } + Logger::MsgBuilder::MsgBuilder( enum LOG_LEVEL msg_lvl, Logger* logger ) + : _msg_lvl( msg_lvl ), _logger(logger) + { + } - logger::sptr logger::_instance = NULL; + Logger::MsgBuilder::MsgBuilder( const Logger::MsgBuilder& rhs ) //: _sink( rhs._sink ) + { + this->_msg_lvl = rhs. _msg_lvl; + _logger = rhs._logger; + } - static logger::sptr _create_logger() - { - logger::sptr the_logger = NULL; + Logger::MsgBuilder::~MsgBuilder() + { + *this << "\n"; + + std::string msg = _oss.str(); - if (!spdlog::get("FDPAPI")) + this->_logger->sink()->execute( _logger, _msg_lvl, msg ); + } + + Logger::MsgBuilder Logger::info() { - the_logger = spdlog::stdout_color_st("FDPAPI"); - } else + return std::move( MsgBuilder( INFO, this ) ); + } + + Logger::MsgBuilder Logger::debug() + { + return std::move( MsgBuilder( DEBUG, this ) ); + } + + Logger::MsgBuilder Logger::trace() + { + return std::move( MsgBuilder( TRACE, this ) ); + } + + Logger::MsgBuilder Logger::warn() + { + return std::move( MsgBuilder( WARN, this ) ); + } + + Logger:: MsgBuilder Logger::critical() { - the_logger = spdlog::get("FDPAPI"); + return std::move( MsgBuilder( WARN, this ) ); } - return the_logger; + Logger::MsgBuilder Logger::error() + { + return std::move( MsgBuilder( ERROR, this ) ); + } + + Sink::Sink( enum LOG_LEVEL log_lvl ) + : ISink(), _log_lvl(log_lvl) + { + _fmtr = SinkFormatter::create(); + } + + enum LOG_LEVEL Sink::log_level(){ return _log_lvl;} + + void Sink::log_level( enum LOG_LEVEL log_lvl ){ _log_lvl = log_lvl;} + + bool Sink::should_log( enum LOG_LEVEL msg_lvl ){ return msg_lvl >= this->log_level();} + + int Sink::execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ) + { + if( this->should_log( msg_lvl ) ) + { + if( NULL != _fmtr ) + { + std::ostringstream oss; + oss << _fmtr->header( msg_lvl, logger ); + + oss << s; + + this->log( msg_lvl, oss.str() ); + } + else + this->log( msg_lvl, s ); + } + return 0; + } + + std::string Sink::apply_formatter( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ) + { + std::ostringstream oss; + + if( NULL != _fmtr ) + { + oss << _fmtr->header( msg_lvl, logger ); + } + oss << msg; + return std::move( oss.str() ); + } + + + std::string SinkFormatter::header( enum LOG_LEVEL msg_lvl, Logger* logger ) + { + std::string dt = curtime(); + std::ostringstream oss; + + oss << "[" << dt << "] "; + + if( !(logger->name().empty()) ) + oss << "[" << logger->name() << "] "; + + + oss << "[" << _LOG_LEVEL_2_str( msg_lvl ) << "] "; + return std::move( oss.str() ); + } + + int CompositeSink::log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + if( this->should_log( msg_lvl ) ) + { + for( int i = 0; i < _sinks.size(); ++i ) + { + Sink::sptr sink = _sinks[i]; + sink->log( msg_lvl, s ); + } + } + + return 0; + } + + + } - logger::sptr logger::get_logger() + class SPDSink : public logging::Sink + { + public: + SPDSink( logging::LOG_LEVEL lvl ) + : logging::Sink( lvl ) + { + //_spd_log = logger::get_spd_logger(); + } + + + private: + //typedef logger::spd_sptr spd_ptr; + //spd_ptr _spd_log; + + + }; + + + logger::logger_sptr logger::_instance = NULL; + + logger::logger_sptr logger::get_logger() { -#if 1 if( NULL == logger::_instance ) { - logger::_instance = _create_logger(); - spdlog::set_default_logger(logger::_instance); + auto sink = logging::OStreamSink::create( logging::LOG_LEVEL::INFO, std::cout ); + logger::_instance = logging::Logger::create( logging::LOG_LEVEL::INFO, sink, "FDPAPI" ); } return logger::_instance; -#else - - static Logger logger( INFO, std::cout ); - return logger; -#endif } + } // namespace FairDataPipeline diff --git a/test/test_api.cxx b/test/test_api.cxx index f31c541..c49bcfb 100644 --- a/test/test_api.cxx +++ b/test/test_api.cxx @@ -39,7 +39,9 @@ class ApiTest : public ::testing::Test { //! [TestGetById] TEST_F(ApiTest, TestGetById) { Json::Value author = api_->get_request(std::string("author/?name=Interface%20Test"))[0]["url"]; - logger::get_logger()->info("Author: {0}", author.asString()); + logger::get_logger()->info() + << "Author: " + << author.asString(); int author_id_ = ApiObject::get_id_from_string( author.asString() ); ASSERT_EQ(api_->get_by_id(std::string("author"), author_id_)["name"].asString(), std::string("Interface Test")); } //! [TestGetById] diff --git a/test/test_config.cxx b/test/test_config.cxx index 6da846b..d12a3fd 100644 --- a/test/test_config.cxx +++ b/test/test_config.cxx @@ -41,7 +41,7 @@ class ConfigTest : public ::testing::Test { ghc::filesystem::path(TESTDIR) / "data" / config; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - logger::get_logger()->set_level(spdlog::level::debug); + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); return Config::construct(config_path_, script_path_, token, use_local ? RESTAPI::LOCAL : RESTAPI::REMOTE ); diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index e5f8b93..35816d8 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -39,8 +39,9 @@ TEST_F(PimplTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "data" / "write_csv.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - logger::get_logger()->set_level(spdlog::level::debug); - DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); + DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token + , logging::LOG_LEVEL::DEBUG ); std::string data_product = "test/csv"; ghc::filesystem::path currentLink = ghc::filesystem::path(dp->link_write(data_product)); @@ -56,7 +57,8 @@ TEST_F(PimplTest, TestDataPipelineInit) { ghc::filesystem::path config_path_read_ = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; - dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); + dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token + , logging::LOG_LEVEL::DEBUG ); currentLink = dp->link_read(data_product); EXPECT_GT(currentLink.string().size(), 1); diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 3f5543a..2d17c3e 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -31,13 +31,13 @@ class RegistryTest : public ::testing::Test { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - logger::get_logger()->set_level(spdlog::level::debug); + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); return DataPipeline::construct( config_path_.string(), script_path_.string(), token, - logger::LOG_LEVEL::log_level_debug ); + logging::LOG_LEVEL::DEBUG ); } std::string token = @@ -51,14 +51,17 @@ TEST_F(RegistryTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - logger::get_logger()->set_level(spdlog::level::debug); + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); - auto dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token, logger::LOG_LEVEL::log_level_debug); + auto dp = DataPipeline::construct( config_path_.string() + , script_path_.string() + , token + , logging::LOG_LEVEL::DEBUG ); } TEST_F(RegistryTest, TestLogLevelSet) { init_pipeline(); - ASSERT_EQ(spdlog::get_level(), spdlog::level::debug); + ASSERT_EQ( logger::get_logger()->get_level(), logging::LOG_LEVEL::DEBUG ); } TEST_F(RegistryTest, TestURLEncode) { @@ -75,10 +78,13 @@ TEST_F(RegistryTest, TestHashFile) { } TEST_F(RegistryTest, TestLogger) { - logging::OStreamSink sink( logging::INFO, std::cout ); - logging::Logger logger( logging::INFO, sink ); - logger.trace() << 1 << " " << "TRACE" << "\n"; - logger.info() << 1 << " " << "INFO" << "\n"; - logger.warn() << 1 << " " << "WARN" << "\n"; + logging::OStreamSink::sptr sink = logging::OStreamSink::create( logging::INFO, std::cout ); + + logging::Logger::sptr logger = logging::Logger::create( logging::INFO, sink ); + logger->trace() << " " << 1 << " " << "TRACE"; + logger->info() << " " << 1 << " " << "INFO"; + logger->warn() << " " << 1 << " " << "WARN"; + + logger::get_logger()->info() << " WARNINF"; //ASSERT_EQ(0,1) ; } From cfd4fcf84431ece3a32ca4a980f94064e579d3bc Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Fri, 1 Apr 2022 14:05:42 +0100 Subject: [PATCH 09/16] Finished logger replacement. Log level specified by environment variable --- include/fdp/fdp.hxx | 8 +-- include/fdp/utilities/logging.hxx | 46 +++++-------- src/fdp.cxx | 52 ++------------ src/utilities/logging.cxx | 108 +++++++++++++++--------------- test/test_pimpl.cxx | 6 +- test/test_registry.cxx | 16 ++--- 6 files changed, 87 insertions(+), 149 deletions(-) diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 00589f6..8d7af42 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -21,13 +21,11 @@ namespace FairDataPipeline { * @param config_file_path * @param script_file_path * @param token - * @param log_level */ static sptr construct( const std::string &config_file_path, const std::string &script_file_path, - std::string token = "", - enum logging::LOG_LEVEL log_level = logging::LOG_LEVEL::INFO ); + std::string token = "" ); /** @@ -70,8 +68,8 @@ namespace FairDataPipeline { explicit DataPipeline( const std::string &config_file_path, const std::string &script_file_path, - std::string token = "", - enum logging::LOG_LEVEL log_level = logging::LOG_LEVEL::INFO); + std::string token = "" + ); DataPipeline(const DataPipeline &rhs) = delete; diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index a610465..88b1b3b 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -10,22 +10,11 @@ ****************************************************************************/ #ifndef __FDP_LOGGING_HXX__ #define __FDP_LOGGING_HXX__ - +#include #include #include #include - -#include "spdlog/spdlog.h" -#if 0 -#define TRACE 0 -#define DEBUG 1 -#define INFO 2 -#define WARN 3 -#define ERROR 4 -#define CRITICAL 5 -#define OFF 6 -#endif - +#include namespace FairDataPipeline { namespace logging { @@ -43,7 +32,7 @@ namespace FairDataPipeline { OFF = 6 }; - const std::string& _LOG_LEVEL_2_str( enum LOG_LEVEL lvl ); + const std::string& to_string( enum LOG_LEVEL lvl ); struct ISink { @@ -57,13 +46,19 @@ namespace FairDataPipeline { class SinkFormatter; typedef std::shared_ptr< SinkFormatter > SinkFormatterPtr; + struct ISinkFormatter + { + typedef std::shared_ptr< ISinkFormatter > sptr; + virtual std::string header(enum LOG_LEVEL msg_lvl, Logger* logger )= 0; + virtual ~ISinkFormatter(){;} + }; + class Sink : public ISink { public: typedef std::shared_ptr< Sink > sptr; - std::string apply_formatter( Logger* loggher, enum LOG_LEVEL msg_level, const std::string& msg ); int execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ); @@ -72,16 +67,17 @@ namespace FairDataPipeline { bool should_log( enum LOG_LEVEL msg_lvl ); - void set_formatter( SinkFormatterPtr fmtr ){_fmtr = fmtr;} + void set_formatter( ISinkFormatter::sptr fmtr ){_fmtr = fmtr;} protected: Sink( enum LOG_LEVEL log_lvl ); - //std::string do_header( enum LOG_LEVEL msg_lvl, Logger* logger ); private: + //std::string apply_formatter( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ); + enum LOG_LEVEL _log_lvl; - SinkFormatterPtr _fmtr; + ISinkFormatter::sptr _fmtr; }; @@ -95,10 +91,8 @@ namespace FairDataPipeline { return sptr( new OStreamSink( lvl, os ) ); } - int log( enum LOG_LEVEL msg_lvl, const std::string& s) { - //if( this->should_log( msg_lvl ) ) _os << s; return 0; @@ -129,11 +123,6 @@ namespace FairDataPipeline { CompositeSink(enum LOG_LEVEL log_lvl) : Sink(log_lvl){ ;} }; - struct ISinkFormatter - { - virtual std::string header(enum LOG_LEVEL msg_lvl, Logger* logger )= 0; - virtual ~ISinkFormatter(){;} - }; class SinkFormatter : public ISinkFormatter { @@ -184,11 +173,7 @@ namespace FairDataPipeline { std::ostringstream _oss; }; - static sptr create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name="" ) - { - return sptr( new Logger(lvl, sink, name)); - } - + static sptr create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name="" ); MsgBuilder info(); MsgBuilder debug(); @@ -210,7 +195,6 @@ namespace FairDataPipeline { _log_lvl = lvl; } - private: enum LOG_LEVEL _log_lvl; diff --git a/src/fdp.cxx b/src/fdp.cxx index 95ee358..5c8c3cd 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -14,22 +14,6 @@ #include "fdp/utilities/logging.hxx" namespace FairDataPipeline { -#if 0 - spdlog::level::level_enum LOG_LEVEL_2_spdlog( logger::LOG_LEVEL fdp_log_level ) - { - std::map< logger::LOG_LEVEL, spdlog::level::level_enum > m; - - m[ logger::LOG_LEVEL::log_level_trace ] = spdlog::level::trace; - m[ logger::LOG_LEVEL::log_level_debug ] = spdlog::level::debug; - m[ logger::LOG_LEVEL::log_level_info ] = spdlog::level::info; - m[ logger::LOG_LEVEL::log_level_warn ] = spdlog::level::warn; - m[ logger::LOG_LEVEL::log_level_error ] = spdlog::level::err; - m[ logger::LOG_LEVEL::log_level_critical ] = spdlog::level::critical; - m[ logger::LOG_LEVEL::log_level_off ] = spdlog::level::off; - - return m[ fdp_log_level ]; - } -#endif /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods @@ -56,7 +40,6 @@ namespace FairDataPipeline { impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - enum logging::LOG_LEVEL log_lvl, RESTAPI api_location = RESTAPI::LOCAL); impl(const impl &dp) = delete; @@ -76,7 +59,6 @@ namespace FairDataPipeline { static sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - enum logging::LOG_LEVEL log_level, RESTAPI api_location = RESTAPI::LOCAL); @@ -126,14 +108,12 @@ namespace FairDataPipeline { DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - enum logging::LOG_LEVEL log_level, RESTAPI api_location) { sptr pobj = impl::sptr( new impl( config_file_path, script_file_path, token, - log_level, api_location ) ); return pobj; } @@ -141,37 +121,18 @@ DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::pa DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - enum logging::LOG_LEVEL log_level, RESTAPI api_location) { - //spdlog::level::level_enum spd_log_lvl = LOG_LEVEL_2_spdlog( log_level ); - this->config_ = Config::construct(config_file_path, script_file_path, token, api_location); const std::string api_root_ = config_->get_api_url(); - -#if 0 - spdlog::set_default_logger(APILogger); - APILogger->set_level( spd_log_lvl ); - - APILogger->info("\n[Configuration]\n\t- Config Path: {0}\n\t- API Root: " - "{1}\n\t- FDP API Token: {2}", - config_file_path.string(), api_root_, - token); -#else - auto the_logger = logger::get_logger(); - //spdlog::set_default_logger(the_logger); - //the_logger->set_level( log_level ); the_logger->info() << "\n[Configuration]\n\t- Config Path:" << config_file_path.string() << "\n\t- API Root: " << api_root_ << "\n\t- FDP API Token: " << token; - -#endif - } ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_read(std::string &data_product){ @@ -193,26 +154,21 @@ DataPipeline::~DataPipeline() = default; DataPipeline::sptr DataPipeline::construct( const std::string &config_file_path, const std::string &script_file_path, - std::string token, - enum logging::LOG_LEVEL log_level ) + std::string token ) { return DataPipeline::sptr( new DataPipeline( config_file_path, script_file_path, - token, - log_level ) ); + token ) ); } DataPipeline::DataPipeline( const std::string &config_file_path, const std::string &script_file_path, - std::string token, - logging::LOG_LEVEL log_level ) -: pimpl_( DataPipeline::impl::construct(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token, - log_level)) + std::string token ) +: pimpl_( DataPipeline::impl::construct(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token )) { - // logger::spd_sptr logger = logger::get_spd_logger(); logger::get_logger()->debug() << "DataPipeline: Initialising session '" << pimpl_->get_code_run_uuid() << "'"; } diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 8c715cd..be1e9e1 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,19 +1,40 @@ -#include "fdp/utilities/logging.hxx" #include +#include #include #include -//#include -//#include -#include "spdlog/sinks/stdout_color_sinks.h" +#include "fdp/utilities/logging.hxx" + namespace FairDataPipeline { namespace logging { - const std::string& _LOG_LEVEL_2_str( enum LOG_LEVEL lvl ) + + const std::string& to_string( enum LOG_LEVEL lvl ) { static const std::string strs[] = {"TRACE","DEBUG","INFO","WARN","ERROR","CRITICAL","OFF"}; return strs[ lvl ]; } + enum LOG_LEVEL to_LOG_LEVEL( const std::string& s ) + { + enum LOG_LEVEL lvl = LOG_LEVEL::OFF; + static std::map< std::string, enum LOG_LEVEL > m = { + {"TRACE", LOG_LEVEL::TRACE } + , {"DEBUG", LOG_LEVEL::DEBUG } + , {"INFO", LOG_LEVEL::INFO } + , {"WARN", LOG_LEVEL::WARN } + , {"ERROR", LOG_LEVEL::ERROR } + , {"CRITICAL", LOG_LEVEL::CRITICAL } + , {"OFF", LOG_LEVEL::OFF } + }; + + auto it = m.find( s ); + + if( it != m.end() ) + lvl = it->second; + + return lvl; + } + std::string curtime() { timeval curTime; @@ -27,6 +48,11 @@ namespace FairDataPipeline { return std::move( currentTime ); } + Logger::sptr Logger::create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) + { + return sptr( new Logger(lvl, sink, name)); + } + Logger::MsgBuilder::MsgBuilder( enum LOG_LEVEL msg_lvl, Logger* logger ) : _msg_lvl( msg_lvl ), _logger(logger) { @@ -40,11 +66,14 @@ namespace FairDataPipeline { Logger::MsgBuilder::~MsgBuilder() { - *this << "\n"; + if( _oss.tellp() > 0 ) + { + *this << "\n"; - std::string msg = _oss.str(); + std::string msg = _oss.str(); - this->_logger->sink()->execute( _logger, _msg_lvl, msg ); + this->_logger->sink()->execute( _logger, _msg_lvl, msg ); + } } Logger::MsgBuilder Logger::info() @@ -89,38 +118,28 @@ namespace FairDataPipeline { bool Sink::should_log( enum LOG_LEVEL msg_lvl ){ return msg_lvl >= this->log_level();} - int Sink::execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ) + int Sink::execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ) { if( this->should_log( msg_lvl ) ) { if( NULL != _fmtr ) { - std::ostringstream oss; - oss << _fmtr->header( msg_lvl, logger ); + //std::ostringstream oss; + //oss << _fmtr->header( msg_lvl, logger ); + //oss << msg; + // + //this->log( msg_lvl, oss.str() ); + std::string s = _fmtr->header(msg_lvl, logger); + s += msg; - oss << s; - - this->log( msg_lvl, oss.str() ); + this->log( msg_lvl, s ); } else - this->log( msg_lvl, s ); + this->log( msg_lvl, msg ); } return 0; } - std::string Sink::apply_formatter( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ) - { - std::ostringstream oss; - - if( NULL != _fmtr ) - { - oss << _fmtr->header( msg_lvl, logger ); - } - oss << msg; - return std::move( oss.str() ); - } - - std::string SinkFormatter::header( enum LOG_LEVEL msg_lvl, Logger* logger ) { std::string dt = curtime(); @@ -131,8 +150,8 @@ namespace FairDataPipeline { if( !(logger->name().empty()) ) oss << "[" << logger->name() << "] "; + oss << "[" << to_string( msg_lvl ) << "] "; - oss << "[" << _LOG_LEVEL_2_str( msg_lvl ) << "] "; return std::move( oss.str() ); } @@ -149,28 +168,8 @@ namespace FairDataPipeline { return 0; } - - - } - class SPDSink : public logging::Sink - { - public: - SPDSink( logging::LOG_LEVEL lvl ) - : logging::Sink( lvl ) - { - //_spd_log = logger::get_spd_logger(); - } - - - private: - //typedef logger::spd_sptr spd_ptr; - //spd_ptr _spd_log; - - - }; - logger::logger_sptr logger::_instance = NULL; @@ -178,13 +177,16 @@ namespace FairDataPipeline { { if( NULL == logger::_instance ) { - auto sink = logging::OStreamSink::create( logging::LOG_LEVEL::INFO, std::cout ); - logger::_instance = logging::Logger::create( logging::LOG_LEVEL::INFO, sink, "FDPAPI" ); + enum logging::LOG_LEVEL log_lvl = logging::OFF; + + if( const char* env = std::getenv( "FDP_LOG_LEVEL" ) ) + log_lvl = logging::to_LOG_LEVEL( env ); + + auto sink = logging::OStreamSink::create( log_lvl, std::cout ); + logger::_instance = logging::Logger::create( log_lvl, sink, "FDPAPI" ); } return logger::_instance; - } - } // namespace FairDataPipeline diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index 35816d8..d5d2a60 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -40,8 +40,7 @@ TEST_F(PimplTest, TestDataPipelineInit) { const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); - DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token - , logging::LOG_LEVEL::DEBUG ); + DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token ); std::string data_product = "test/csv"; ghc::filesystem::path currentLink = ghc::filesystem::path(dp->link_write(data_product)); @@ -57,8 +56,7 @@ TEST_F(PimplTest, TestDataPipelineInit) { ghc::filesystem::path config_path_read_ = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; - dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token - , logging::LOG_LEVEL::DEBUG ); + dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token ); currentLink = dp->link_read(data_product); EXPECT_GT(currentLink.string().size(), 1); diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 2d17c3e..c672aee 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -31,13 +31,13 @@ class RegistryTest : public ::testing::Test { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); return DataPipeline::construct( config_path_.string(), script_path_.string(), - token, - logging::LOG_LEVEL::DEBUG ); + token ); } std::string token = @@ -51,12 +51,12 @@ TEST_F(RegistryTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); auto dp = DataPipeline::construct( config_path_.string() , script_path_.string() - , token - , logging::LOG_LEVEL::DEBUG ); + , token ); } TEST_F(RegistryTest, TestLogLevelSet) { @@ -81,10 +81,10 @@ TEST_F(RegistryTest, TestLogger) { logging::OStreamSink::sptr sink = logging::OStreamSink::create( logging::INFO, std::cout ); logging::Logger::sptr logger = logging::Logger::create( logging::INFO, sink ); - logger->trace() << " " << 1 << " " << "TRACE"; - logger->info() << " " << 1 << " " << "INFO"; - logger->warn() << " " << 1 << " " << "WARN"; + logger->trace() << " " << 10 << " " << "TRACE"; + logger->info() << " " << 50 << " " << "INFO"; + + logger->warn() << " " << 100 << " " << "WARN"; logger::get_logger()->info() << " WARNINF"; - //ASSERT_EQ(0,1) ; } From 0fc6613170c2e3748e1f57b341d298a1429373f7 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Tue, 5 Apr 2022 12:26:55 +0100 Subject: [PATCH 10/16] remove spd header --- include/fdp/fdp.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 8d7af42..43e5048 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -1,8 +1,8 @@ #ifndef __FDP__ #define __FDP__ -//#include "spdlog/spdlog.h" #include "utilities/logging.hxx" + #include namespace FairDataPipeline { From abbc04f1cb57e128988727fb28598205104d4af1 Mon Sep 17 00:00:00 2001 From: Derek Sandiford Date: Wed, 6 Apr 2022 10:00:01 +0100 Subject: [PATCH 11/16] logging: tidy up --- include/fdp/fdp.hxx | 1 - include/fdp/utilities/logging.hxx | 27 ++++++--------------------- src/fdp.cxx | 1 - src/utilities/logging.cxx | 25 ++++++++++++++++++++++++- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 8d7af42..d45e349 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -1,7 +1,6 @@ #ifndef __FDP__ #define __FDP__ -//#include "spdlog/spdlog.h" #include "utilities/logging.hxx" #include diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index 88b1b3b..1329887 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -19,8 +19,6 @@ namespace FairDataPipeline { namespace logging { - std::string curtime(); - enum LOG_LEVEL { TRACE = 0, @@ -60,21 +58,20 @@ namespace FairDataPipeline { typedef std::shared_ptr< Sink > sptr; - int execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ); - enum LOG_LEVEL log_level(); void log_level( enum LOG_LEVEL log_lvl ); bool should_log( enum LOG_LEVEL msg_lvl ); void set_formatter( ISinkFormatter::sptr fmtr ){_fmtr = fmtr;} + + int execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ); protected: Sink( enum LOG_LEVEL log_lvl ); private: - //std::string apply_formatter( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ); enum LOG_LEVEL _log_lvl; ISinkFormatter::sptr _fmtr; @@ -86,21 +83,13 @@ namespace FairDataPipeline { public: typedef std::shared_ptr< OStreamSink > sptr; - static sptr create( enum LOG_LEVEL lvl, std::ostream& os ) - { - return sptr( new OStreamSink( lvl, os ) ); - } - - int log( enum LOG_LEVEL msg_lvl, const std::string& s) - { - _os << s; + static sptr create( enum LOG_LEVEL lvl, std::ostream& os ); - return 0; - } + int log( enum LOG_LEVEL msg_lvl, const std::string& s); private: - OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os){ ;} + OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os); std::ostream& _os; }; @@ -189,11 +178,7 @@ namespace FairDataPipeline { enum LOG_LEVEL get_level() const { return _log_lvl;} protected: - Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) - : _sink(sink), _name(name) - { - _log_lvl = lvl; - } + Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) ; private: diff --git a/src/fdp.cxx b/src/fdp.cxx index 5c8c3cd..b383acd 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -4,7 +4,6 @@ #include #include -#include "spdlog/spdlog.h" #include "json/json.h" #include "fdp/objects/config.hxx" diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index be1e9e1..2d350bd 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -35,7 +35,7 @@ namespace FairDataPipeline { return lvl; } - std::string curtime() + static std::string curtime() { timeval curTime; gettimeofday( &curTime, NULL ); @@ -48,6 +48,12 @@ namespace FairDataPipeline { return std::move( currentTime ); } + Logger::Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) + : _sink(sink), _name(name) + { + _log_lvl = lvl; + } + Logger::sptr Logger::create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) { return sptr( new Logger(lvl, sink, name)); @@ -140,6 +146,23 @@ namespace FairDataPipeline { return 0; } + OStreamSink::sptr OStreamSink::create( enum LOG_LEVEL lvl, std::ostream& os ) + { + return sptr( new OStreamSink( lvl, os ) ); + } + + int OStreamSink::log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + _os << s; + + return 0; + } + + OStreamSink::OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os) + { + } + + std::string SinkFormatter::header( enum LOG_LEVEL msg_lvl, Logger* logger ) { std::string dt = curtime(); From 41dcbbd61ae977ddb9136d86c6cb3b8335effc9c Mon Sep 17 00:00:00 2001 From: Derek Sandiford <64950361+dezed-ukaea@users.noreply.github.com> Date: Wed, 6 Apr 2022 13:09:32 +0100 Subject: [PATCH 12/16] Update README.md Add instructions on specifying logging level. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3905243..0c8cf4c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ $ cmake --build build --config=Release ## Outline The main class the user will interact with is `DataPipeline` which has only the required methods such as `link_read` etc. This class has a member which is a pointer to an underlying `DataPipelineImpl_` class which performs the various procedures required to handle the data. A logger has been used to give as much feedback to the user as possible, the verbosity being handled by a log level argument. +### Logging +The environment variable `FDP_LOG_LEVEL=[TRACE:DEBUG:INFO:WARN:ERROR:CRITICAL:OFF]` can be set to specify the logging output level. ## Unit Tests The unit tests use the local registry, this needs to be running prior to running the tests see: [the CLI documentation](https://github.com/FAIRDataPipeline/FAIR-CLI#registry) From d72c1d947beb8cc65308024db999aed74ab3124a Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Sun, 24 Apr 2022 06:45:25 +0100 Subject: [PATCH 13/16] Remove SPDLOG from dependancies, fix log windows --- CMakeLists.txt | 1 - external/digestpp.cmake | 1 + external/jsoncpp.cmake | 2 ++ external/spdlog.cmake | 13 ------------- include/sys/time.cpp | 18 ++++++++++++++++++ include/sys/time.h | 7 +++++++ include/sys/times.h | 38 ++++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 5 ++--- src/utilities/logging.cxx | 3 ++- 9 files changed, 70 insertions(+), 18 deletions(-) delete mode 100644 external/spdlog.cmake create mode 100644 include/sys/time.cpp create mode 100644 include/sys/time.h create mode 100644 include/sys/times.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 40d0da3..be7034f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,6 @@ INCLUDE( external/jsoncpp.cmake ) INCLUDE( external/digestpp.cmake ) INCLUDE( external/curl.cmake ) INCLUDE( external/yaml_cpp.cmake ) -INCLUDE( external/spdlog.cmake ) INCLUDE( external/toml11.cmake ) INCLUDE( external/ghc.cmake ) diff --git a/external/digestpp.cmake b/external/digestpp.cmake index 211d805..7040371 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -7,6 +7,7 @@ SET( DIGESTCPP_COMMIT "34ff2eeae397ed744d972d86b5a20f603b029fbd" ) # So tell the compiler to exclude min and max macros in windows IF(WIN32) add_definitions(-DNOMINMAX) + add_definitions(-DNOGDI) ENDIF() MESSAGE( STATUS "[DigestCPP]" ) diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index 09cda58..41f6836 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -4,6 +4,8 @@ MESSAGE( STATUS "[JsonCPP]" ) MESSAGE( STATUS "\tJsonCpp Will be installed." ) MESSAGE( STATUS "\tURL: ${JSONCPP_URL}" ) +SET (JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't Build cpp tests") + include(FetchContent) FetchContent_Declare( JsonCpp diff --git a/external/spdlog.cmake b/external/spdlog.cmake deleted file mode 100644 index 9765dad..0000000 --- a/external/spdlog.cmake +++ /dev/null @@ -1,13 +0,0 @@ -MESSAGE( STATUS "[SPD Log]" ) - -set(SPDLOG_URL "https://github.com/gabime/spdlog/archive/refs/tags/v1.9.2.zip") - -MESSAGE( STATUS "\tSPD Will be installed." ) -MESSAGE( STATUS "\tURL: ${SPDLOG_URL}" ) - -include(FetchContent) -FetchContent_Declare( - SPDLOG - URL ${SPDLOG_URL} -) -FetchContent_MakeAvailable(SPDLOG) \ No newline at end of file diff --git a/include/sys/time.cpp b/include/sys/time.cpp new file mode 100644 index 0000000..9cd9d5a --- /dev/null +++ b/include/sys/time.cpp @@ -0,0 +1,18 @@ +#include "sys/times.h" + +int gettimeofday(struct timeval* t,void* timezone) +{ struct _timeb timebuffer; + _ftime( &timebuffer ); + t->tv_sec=timebuffer.time; + t->tv_usec=1000*timebuffer.millitm; + return 0; +} + +clock_t times (struct tms *__buffer) { + + __buffer->tms_utime = clock(); + __buffer->tms_stime = 0; + __buffer->tms_cstime = 0; + __buffer->tms_cutime = 0; + return __buffer->tms_utime; +} \ No newline at end of file diff --git a/include/sys/time.h b/include/sys/time.h new file mode 100644 index 0000000..6599d63 --- /dev/null +++ b/include/sys/time.h @@ -0,0 +1,7 @@ +#pragma once + +#ifndef _TIMES_H + +#include "sys/times.h" + +#endif \ No newline at end of file diff --git a/include/sys/times.h b/include/sys/times.h new file mode 100644 index 0000000..bd3cade --- /dev/null +++ b/include/sys/times.h @@ -0,0 +1,38 @@ +#ifndef _TIMES_H +#define _TIMES_H + +#ifdef _WIN32 +#include +#include +#include + +int gettimeofday(struct timeval* t,void* timezone); + +// from linux's sys/times.h + +//#include + +#define __need_clock_t +#include + + +/* Structure describing CPU time used by a process and its children. */ +struct tms + { + clock_t tms_utime; /* User CPU time. */ + clock_t tms_stime; /* System CPU time. */ + + clock_t tms_cutime; /* User CPU time of dead children. */ + clock_t tms_cstime; /* System CPU time of dead children. */ + }; + +/* Store the CPU time used by this process and all its + dead children (and their dead children) in BUFFER. + Return the elapsed real time, or (clock_t) -1 for errors. + All times are in CLK_TCKths of a second. */ +clock_t times (struct tms *__buffer); + +typedef long long suseconds_t ; + +#endif +#endif \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 538cf4d..c1040d4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ # Find and add the .cxx files to SRC_FILES -FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx ) +FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx +${FDPAPI_INCLUDE_DIRS}/sys/*.cpp ) # Add the (Static) Project Library using SRC_FILES ADD_LIBRARY( ${FDPAPI} STATIC ${SRC_FILES} ) @@ -16,7 +17,6 @@ TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${jsoncpp_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${yamlcpp_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${CURL_INCLUDE_DIRS} ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${digestpp_SOURCE_DIR} ) -TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${spdlog_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${ghc_SOURCE_DIR}/include ) # Toml 11 is a header only library no need to link TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${tomlcpp_SOURCE_DIR} ) @@ -25,7 +25,6 @@ TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${tomlcpp_SOURCE_DIR} ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC jsoncpp_static ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC digestpp ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC yaml-cpp ) -TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC spdlog ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC ghc_filesystem ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC ${CURL_LIBRARIES} ) diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 2d350bd..1b521dc 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -41,7 +41,8 @@ namespace FairDataPipeline { gettimeofday( &curTime, NULL ); int milli = curTime.tv_usec / 1000; char buffer[ 80 ]; - strftime( buffer, 80, "%Y-%m-%d %H:%M:%S", localtime( &curTime.tv_sec )); + std::time_t _tv_sec = (time_t)curTime.tv_sec; + strftime( buffer, 80, "%Y-%m-%d %H:%M:%S", localtime( &_tv_sec )); std::string currentTime( 84, 0 ); sprintf( ¤tTime[0], "%s.%03d", buffer, milli ); From d31f3fbf68dd446b70d47b4e14bc6e44f9de1504 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Sun, 24 Apr 2022 06:55:54 +0100 Subject: [PATCH 14/16] Only include time.cpp on windows --- src/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c1040d4..97eba6e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,9 @@ # Find and add the .cxx files to SRC_FILES -FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx -${FDPAPI_INCLUDE_DIRS}/sys/*.cpp ) +FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx) + +IF(WIN32) + list(APPEND SRC_FILES ${FDPAPI_INCLUDE_DIRS}/sys/time.cpp ) +ENDIF() # Add the (Static) Project Library using SRC_FILES ADD_LIBRARY( ${FDPAPI} STATIC ${SRC_FILES} ) From 8c85b2130f3c5da4947779e512f061d317b8d629 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Sun, 24 Apr 2022 07:10:19 +0100 Subject: [PATCH 15/16] fix time includes --- include/{sys => windows_sys}/time.cpp | 2 +- include/{sys => windows_sys}/time.h | 2 +- include/{sys => windows_sys}/times.h | 0 src/CMakeLists.txt | 2 +- src/utilities/logging.cxx | 6 +++++- 5 files changed, 8 insertions(+), 4 deletions(-) rename include/{sys => windows_sys}/time.cpp (92%) rename include/{sys => windows_sys}/time.h (55%) rename include/{sys => windows_sys}/times.h (100%) diff --git a/include/sys/time.cpp b/include/windows_sys/time.cpp similarity index 92% rename from include/sys/time.cpp rename to include/windows_sys/time.cpp index 9cd9d5a..845a90a 100644 --- a/include/sys/time.cpp +++ b/include/windows_sys/time.cpp @@ -1,4 +1,4 @@ -#include "sys/times.h" +#include "windows_sys/times.h" int gettimeofday(struct timeval* t,void* timezone) { struct _timeb timebuffer; diff --git a/include/sys/time.h b/include/windows_sys/time.h similarity index 55% rename from include/sys/time.h rename to include/windows_sys/time.h index 6599d63..adc4637 100644 --- a/include/sys/time.h +++ b/include/windows_sys/time.h @@ -2,6 +2,6 @@ #ifndef _TIMES_H -#include "sys/times.h" +#include "windows_sys/times.h" #endif \ No newline at end of file diff --git a/include/sys/times.h b/include/windows_sys/times.h similarity index 100% rename from include/sys/times.h rename to include/windows_sys/times.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97eba6e..ee84461 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,7 @@ FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx) IF(WIN32) - list(APPEND SRC_FILES ${FDPAPI_INCLUDE_DIRS}/sys/time.cpp ) + list(APPEND SRC_FILES ${FDPAPI_INCLUDE_DIRS}/windows_sys/time.cpp ) ENDIF() # Add the (Static) Project Library using SRC_FILES diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 1b521dc..59609b4 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,6 +1,10 @@ #include #include -#include +#ifdef _WIN32 + #include +#else + #include +#endif #include #include "fdp/utilities/logging.hxx" From 91751495de53777409065b09bccc06d78e340f84 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Sun, 24 Apr 2022 07:21:52 +0100 Subject: [PATCH 16/16] explicitly include ctime --- src/utilities/logging.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 59609b4..6640dc9 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -6,6 +6,7 @@ #include #endif #include +#include #include "fdp/utilities/logging.hxx" @@ -45,7 +46,7 @@ namespace FairDataPipeline { gettimeofday( &curTime, NULL ); int milli = curTime.tv_usec / 1000; char buffer[ 80 ]; - std::time_t _tv_sec = (time_t)curTime.tv_sec; + std::time_t _tv_sec = curTime.tv_sec; strftime( buffer, 80, "%Y-%m-%d %H:%M:%S", localtime( &_tv_sec )); std::string currentTime( 84, 0 );