diff --git a/CMakeLists.txt b/CMakeLists.txt index 40d0da3..be7034f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,6 @@ INCLUDE( external/jsoncpp.cmake ) INCLUDE( external/digestpp.cmake ) INCLUDE( external/curl.cmake ) INCLUDE( external/yaml_cpp.cmake ) -INCLUDE( external/spdlog.cmake ) INCLUDE( external/toml11.cmake ) INCLUDE( external/ghc.cmake ) diff --git a/README.md b/README.md index 3905243..0c8cf4c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ $ cmake --build build --config=Release ## Outline The main class the user will interact with is `DataPipeline` which has only the required methods such as `link_read` etc. This class has a member which is a pointer to an underlying `DataPipelineImpl_` class which performs the various procedures required to handle the data. A logger has been used to give as much feedback to the user as possible, the verbosity being handled by a log level argument. +### Logging +The environment variable `FDP_LOG_LEVEL=[TRACE:DEBUG:INFO:WARN:ERROR:CRITICAL:OFF]` can be set to specify the logging output level. ## Unit Tests The unit tests use the local registry, this needs to be running prior to running the tests see: [the CLI documentation](https://github.com/FAIRDataPipeline/FAIR-CLI#registry) diff --git a/external/digestpp.cmake b/external/digestpp.cmake index 211d805..7040371 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -7,6 +7,7 @@ SET( DIGESTCPP_COMMIT "34ff2eeae397ed744d972d86b5a20f603b029fbd" ) # So tell the compiler to exclude min and max macros in windows IF(WIN32) add_definitions(-DNOMINMAX) + add_definitions(-DNOGDI) ENDIF() MESSAGE( STATUS "[DigestCPP]" ) diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index 09cda58..41f6836 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -4,6 +4,8 @@ MESSAGE( STATUS "[JsonCPP]" ) MESSAGE( STATUS "\tJsonCpp Will be installed." ) MESSAGE( STATUS "\tURL: ${JSONCPP_URL}" ) +SET (JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't Build cpp tests") + include(FetchContent) FetchContent_Declare( JsonCpp diff --git a/external/spdlog.cmake b/external/spdlog.cmake deleted file mode 100644 index 9765dad..0000000 --- a/external/spdlog.cmake +++ /dev/null @@ -1,13 +0,0 @@ -MESSAGE( STATUS "[SPD Log]" ) - -set(SPDLOG_URL "https://github.com/gabime/spdlog/archive/refs/tags/v1.9.2.zip") - -MESSAGE( STATUS "\tSPD Will be installed." ) -MESSAGE( STATUS "\tURL: ${SPDLOG_URL}" ) - -include(FetchContent) -FetchContent_Declare( - SPDLOG - URL ${SPDLOG_URL} -) -FetchContent_MakeAvailable(SPDLOG) \ No newline at end of file diff --git a/include/fdp/exceptions.hxx b/include/fdp/exceptions.hxx index 9c379e2..ff2cd60 100644 --- a/include/fdp/exceptions.hxx +++ b/include/fdp/exceptions.hxx @@ -4,7 +4,7 @@ #include #include -namespace FDP { +namespace FairDataPipeline { class config_parsing_error : public std::runtime_error { public: config_parsing_error(const std::string& message) @@ -37,6 +37,6 @@ public: write_error(const std::string& message) : std::runtime_error(message) {} }; -}; // namespace FDP +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/fdp.hxx b/include/fdp/fdp.hxx index 7a329e3..43e5048 100644 --- a/include/fdp/fdp.hxx +++ b/include/fdp/fdp.hxx @@ -1,12 +1,11 @@ #ifndef __FDP__ #define __FDP__ -#include "spdlog/spdlog.h" -#include +#include "utilities/logging.hxx" -//#include "fdp/registry/datapipeline.hxx" +#include -namespace FDP { +namespace FairDataPipeline { /** * @brief DataPipeline Class: * A PIMPL Class for interacting the the FAIR Data Pipeline @@ -15,19 +14,19 @@ namespace FDP { class DataPipeline { public: + typedef std::shared_ptr< DataPipeline > sptr; /** * @brief Construct a new Data Pipeline (PIMPL) * * @param config_file_path * @param script_file_path * @param token - * @param log_level */ - explicit DataPipeline( + static sptr construct( const std::string &config_file_path, const std::string &script_file_path, - std::string token = "", - spdlog::level::level_enum log_level = spdlog::level::info); + std::string token = "" ); + /** * @brief Destroy the Data Pipeline object @@ -35,11 +34,9 @@ namespace FDP { */ ~DataPipeline(); - DataPipeline(DataPipeline &&rhs) noexcept; - DataPipeline &operator=(DataPipeline &&rhs) noexcept; + //DataPipeline(DataPipeline &&rhs) noexcept; + //DataPipeline &operator=(DataPipeline &&rhs) noexcept; - DataPipeline(const DataPipeline &rhs); - DataPipeline &operator=(const DataPipeline &rhs); /** * @brief Return a path to a given data product @@ -68,10 +65,19 @@ namespace FDP { void finalise(); private: + explicit DataPipeline( + const std::string &config_file_path, + const std::string &script_file_path, + std::string token = "" + ); + + DataPipeline(const DataPipeline &rhs) = delete; + + DataPipeline &operator=(const DataPipeline &rhs) = delete; + class impl; std::shared_ptr< DataPipeline::impl > pimpl_; }; -}; // namespace FDP - +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/objects/api_object.hxx b/include/fdp/objects/api_object.hxx index b221402..6069c3c 100644 --- a/include/fdp/objects/api_object.hxx +++ b/include/fdp/objects/api_object.hxx @@ -7,7 +7,7 @@ #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace FairDataPipeline { /** * @brief Class for API objects * @@ -49,7 +49,7 @@ namespace FDP { * @return int object id e.g. if the uri is http://127.0.0.1/object/1 * the object id will be 1 */ - int get_id(); + int get_id() const ; /** * @brief Get the object type from the uri * @@ -58,7 +58,7 @@ namespace FDP { */ static int get_id_from_string(std::string url); - std::string get_type(); + std::string get_type() const ; /** * @brief Get the object uri * diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index d2acb90..e163e54 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -25,12 +25,17 @@ #include "fdp/objects/api_object.hxx" #include "fdp/objects/io_object.hxx" -namespace FDP { +namespace FairDataPipeline { /** * @brief class for interacting with confifurations * */ class Config { + public: + typedef std::shared_ptr< Config > sptr; + + typedef std::map< std::string, IOObject > map_type; + private: const ghc::filesystem::path config_file_path_; const ghc::filesystem::path config_dir_; @@ -59,11 +64,11 @@ namespace FDP { ApiObject::sptr code_run_; - std::map writes_; - std::map reads_; + map_type writes_; + map_type reads_; - std::map outputs_; - std::map inputs_; + map_type outputs_; + map_type inputs_; RESTAPI rest_api_location_ = RESTAPI::LOCAL; @@ -87,7 +92,6 @@ namespace FDP { public: - typedef std::shared_ptr< Config > sptr; static Config::sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, @@ -225,7 +229,7 @@ namespace FDP { * @param data_product * @return ghc::filesystem::path */ - ghc::filesystem::path link_write(std::string &data_product); + ghc::filesystem::path link_write( const std::string &data_product); /** * @brief Return the filepath to a given data product @@ -233,7 +237,7 @@ namespace FDP { * @param data_product * @return ghc::filesystem::path */ - ghc::filesystem::path link_read(std::string &data_product); + ghc::filesystem::path link_read(const std::string& data_product); /** * @brief Finalise the pipeline diff --git a/include/fdp/objects/io_object.hxx b/include/fdp/objects/io_object.hxx index 19bb5c0..0ea5314 100644 --- a/include/fdp/objects/io_object.hxx +++ b/include/fdp/objects/io_object.hxx @@ -8,7 +8,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/objects/api_object.hxx" -namespace FDP { +namespace FairDataPipeline { /** * @brief Class for API objects * @@ -123,7 +123,7 @@ namespace FDP { * * @return const std::string& */ - const std::string& get_use_namespace() {return use_namespace_;} + const std::string& get_use_namespace() const {return use_namespace_;} /** * @brief Get the path of the data product @@ -144,7 +144,7 @@ namespace FDP { * * @return std::string */ - const std::string& get_component_description() {return component_description_;} + const std::string& get_component_description() const {return component_description_;} /** * @brief Check whether the data product is public diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index 299703d..83c43dd 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/objects/metadata.hxx + * @file FairDataPipeline/objects/metadata.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-05 * @brief File containing methods for handling of metadata @@ -29,7 +29,7 @@ #include #endif -namespace FDP { +namespace FairDataPipeline { /*! ************************************************************************** * @brief calculates a hash from a given input file via SHA1 * @@ -94,6 +94,6 @@ bool file_exists( const std::string &Filename ); */ std::string read_token(const ghc::filesystem::path &token_path); -}; // namespace FDP +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index b5d8e37..94692a5 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -1,10 +1,10 @@ /*! ************************************************************************** - * @file fdp/registry/api.hxx + * @file FairDataPipeline/registry/api.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-05 * @brief File containing classes and methods for connecting to the RestAPI * - * The classes and methods within this file are used to access the FDP + * The classes and methods within this file are used to access the FairDataPipeline * Data Pipeline RestAPI, they handle sending of requests and retrieval of * data as JSON strings ****************************************************************************/ @@ -27,7 +27,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace FairDataPipeline { /*! ************************************************************************** * @enum RESTAPI * @brief selection of either local or remote pipeline running @@ -39,6 +39,7 @@ enum class RESTAPI { LOCAL /*!< Run from local registry */ }; +#if 0 /*! ************************************************************************** * @brief function for writing to file from a CURL call * @author K. Zarebski (UKAEA) @@ -50,6 +51,7 @@ enum class RESTAPI { * @return size_t ****************************************************************************/ size_t write_func_(void *ptr, size_t size, size_t nmemb, std::string *data); +#endif /*! ************************************************************************** * @class API @@ -70,8 +72,8 @@ public: * * @param url_root the root of the query address, e.g. localhost:8000/api ***************************************************************************/ - API(std::string url_root) - : url_root_(API::append_with_forward_slash(url_root)) {} + static sptr construct( const std::string& url_root ); + /** * @brief sends the given 'packet' of information to the RestAPI @@ -164,6 +166,9 @@ public: static std::string remove_leading_forward_slash(std::string str); private: + API( const std::string& url_root) + : url_root_(API::append_with_forward_slash(url_root)) {} + std::string url_root_; CURL *setup_json_session_(std::string &addr_path, std::string *response, long &http_code, std::string token = ""); @@ -181,8 +186,8 @@ private: ghc::filesystem::path out_path); }; -std::string url_encode(std::string url); +std::string url_encode(const std::string& url); -}; // namespace FDP +}; // namespace FairDataPipeline #endif diff --git a/include/fdp/registry/data_io.hxx b/include/fdp/registry/data_io.hxx index 5d1bd15..60f47c0 100644 --- a/include/fdp/registry/data_io.hxx +++ b/include/fdp/registry/data_io.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/registry/config.hxx + * @file FairDataPipeline/registry/config.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File containing classes for interacting with the local file system @@ -27,7 +27,7 @@ #include "fdp/utilities/logging.hxx" #include "fdp/utilities/semver.hxx" -namespace FDP { +namespace FairDataPipeline { /*! *************************************************************************** * @brief read the value of a point estimate from a given TOML file @@ -87,8 +87,10 @@ ghc::filesystem::path create_estimate(T &value, toml_out_.close(); - APILogger->debug("FileSystem:CreateEstimate: Wrote point estimate to '{0}'", - output_filename_.string()); + + auto the_logger = logger::get_logger(); + the_logger->debug() + << "FileSystem:CreateEstimate: Wrote point estimate to '" << output_filename_.string() << "'"; return output_filename_; } @@ -101,6 +103,6 @@ ghc::filesystem::path create_estimate(T &value, */ std::string get_first_key_(const toml::value data_table); -}; // namespace FDP +}; // namespace FairDataPipeline -#endif \ No newline at end of file +#endif diff --git a/include/fdp/utilities/json.hxx b/include/fdp/utilities/json.hxx index ed4e687..6f5e369 100644 --- a/include/fdp/utilities/json.hxx +++ b/include/fdp/utilities/json.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/json.hxx + * @file FairDataPipeline/utilities/json.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains methods relating to handling of JSON based data @@ -12,7 +12,7 @@ #include -namespace FDP { +namespace FairDataPipeline { /*! ************************************************************************** * @brief convert a JSON object to string form * @author K. Zarebski (UKAEA) @@ -29,6 +29,6 @@ namespace FDP { * ****************************************************************************/ std::string json_to_string(Json::Value &json_data); -}; // namespace FDP +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/fdp/utilities/logging.hxx b/include/fdp/utilities/logging.hxx index 9a8e504..1329887 100644 --- a/include/fdp/utilities/logging.hxx +++ b/include/fdp/utilities/logging.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/logging.hxx + * @file FairDataPipeline/utilities/logging.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains pointers for creating global logger during session @@ -10,26 +10,195 @@ ****************************************************************************/ #ifndef __FDP_LOGGING_HXX__ #define __FDP_LOGGING_HXX__ +#include +#include +#include +#include +#include -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/spdlog.h" +namespace FairDataPipeline { + namespace logging { -namespace FDP { + enum LOG_LEVEL + { + TRACE = 0, + DEBUG = 1, + INFO = 2, + WARN = 3, + ERROR = 4, + CRITICAL = 5, + OFF = 6 + }; + + const std::string& to_string( enum LOG_LEVEL lvl ); + + struct ISink + { + typedef std::shared_ptr< ISink > sptr; + virtual int log( enum LOG_LEVEL msg_lvl, const std::string& msg ) = 0; + virtual ~ISink(){}; + }; + + class Logger; + + class SinkFormatter; + typedef std::shared_ptr< SinkFormatter > SinkFormatterPtr; + + struct ISinkFormatter + { + typedef std::shared_ptr< ISinkFormatter > sptr; + virtual std::string header(enum LOG_LEVEL msg_lvl, Logger* logger )= 0; + virtual ~ISinkFormatter(){;} + }; + + + class Sink : public ISink + { + public: + typedef std::shared_ptr< Sink > sptr; + + + enum LOG_LEVEL log_level(); + void log_level( enum LOG_LEVEL log_lvl ); + + bool should_log( enum LOG_LEVEL msg_lvl ); + + void set_formatter( ISinkFormatter::sptr fmtr ){_fmtr = fmtr;} + + int execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& s ); + + protected: + Sink( enum LOG_LEVEL log_lvl ); + + private: + + + enum LOG_LEVEL _log_lvl; + ISinkFormatter::sptr _fmtr; + }; + + + class OStreamSink : public Sink + { + public: + typedef std::shared_ptr< OStreamSink > sptr; + + static sptr create( enum LOG_LEVEL lvl, std::ostream& os ); + + int log( enum LOG_LEVEL msg_lvl, const std::string& s); + + private: + + OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os); + + std::ostream& _os; + }; + + class CompositeSink : public Sink + { + public: + typedef std::shared_ptr< CompositeSink > sptr; + + + static sptr create( enum LOG_LEVEL lvl ){ return sptr( new CompositeSink( lvl ) ); } + + + int log( enum LOG_LEVEL msg_lvl, const std::string& s); + void add_sink( Sink::sptr sink ){ _sinks.push_back( sink ); } + + private: + std::vector< Sink::sptr > _sinks; + + CompositeSink(enum LOG_LEVEL log_lvl) : Sink(log_lvl){ ;} + }; + + + class SinkFormatter : public ISinkFormatter + { + public: + typedef std::shared_ptr< SinkFormatter > sptr; + + static sptr create(){ return sptr( new SinkFormatter() ); } + + ~SinkFormatter() + { + } + + std::string header( enum LOG_LEVEL msg_lvl, Logger* logger ); + + private: + SinkFormatter(){} + }; + + class Logger + { + public: + typedef std::shared_ptr< Logger > sptr; + class MsgBuilder + { + public: + friend class Logger; + typedef std::shared_ptr< MsgBuilder > sptr; + + ~MsgBuilder(); + + template< typename T > + MsgBuilder& operator<<( const T& s) + { + if( _logger->sink()->should_log( _msg_lvl ) ) + { + _oss << s; + } + return *this; + } + + private: + MsgBuilder( enum LOG_LEVEL msg_lvl, Logger* logger ); + MsgBuilder( const MsgBuilder& rhs ); + + Logger* _logger; + enum LOG_LEVEL _msg_lvl; + + std::ostringstream _oss; + }; + + static sptr create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name="" ); + + MsgBuilder info(); + MsgBuilder debug(); + MsgBuilder trace(); + MsgBuilder warn(); + MsgBuilder critical(); + MsgBuilder error(); + + Sink::sptr sink(){ return _sink; } + const std::string& name() const { return _name;} + + void set_level( enum LOG_LEVEL lvl){ _log_lvl = lvl;} + enum LOG_LEVEL get_level() const { return _log_lvl;} + + protected: + Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) ; + + private: + + enum LOG_LEVEL _log_lvl; + Sink::sptr _sink; + std::string _name; + }; + } class logger { public: - typedef std::shared_ptr sptr; + typedef std::shared_ptr< logging::Logger > logger_sptr; + + static logger_sptr get_logger(); private: + static logger_sptr _instance; }; - /** - * @brief Global logger used across all FDP scripts - * @author K. Zarebski (UKAEA) - * - */ - extern const logger::sptr APILogger; -} // namespace FDP +} // namespace FairDataPipeline diff --git a/include/fdp/utilities/semver.hxx b/include/fdp/utilities/semver.hxx index 27936fd..8b2e904 100644 --- a/include/fdp/utilities/semver.hxx +++ b/include/fdp/utilities/semver.hxx @@ -1,5 +1,5 @@ /*! ************************************************************************** - * @file fdp/utilities/semver.hxx + * @file FairDataPipeline/utilities/semver.hxx * @author K. Zarebski (UKAEA) * @date 2021-05-06 * @brief File contains classes and types for validating semantic versioning @@ -16,9 +16,9 @@ #include #include -namespace FDP { +namespace FairDataPipeline { /*! ************************************************************************** - * @namespace FDP::Versioning + * @namespace FairDataPipeline::Versioning * @brief namespace containing all semantic versioning types and classes * @author K. Zarebski (UKAEA) * @@ -199,6 +199,6 @@ private: std::string get_tag_str() const; }; }; // namespace Versioning -}; // namespace FDP +}; // namespace FairDataPipeline #endif \ No newline at end of file diff --git a/include/windows_sys/time.cpp b/include/windows_sys/time.cpp new file mode 100644 index 0000000..845a90a --- /dev/null +++ b/include/windows_sys/time.cpp @@ -0,0 +1,18 @@ +#include "windows_sys/times.h" + +int gettimeofday(struct timeval* t,void* timezone) +{ struct _timeb timebuffer; + _ftime( &timebuffer ); + t->tv_sec=timebuffer.time; + t->tv_usec=1000*timebuffer.millitm; + return 0; +} + +clock_t times (struct tms *__buffer) { + + __buffer->tms_utime = clock(); + __buffer->tms_stime = 0; + __buffer->tms_cstime = 0; + __buffer->tms_cutime = 0; + return __buffer->tms_utime; +} \ No newline at end of file diff --git a/include/windows_sys/time.h b/include/windows_sys/time.h new file mode 100644 index 0000000..adc4637 --- /dev/null +++ b/include/windows_sys/time.h @@ -0,0 +1,7 @@ +#pragma once + +#ifndef _TIMES_H + +#include "windows_sys/times.h" + +#endif \ No newline at end of file diff --git a/include/windows_sys/times.h b/include/windows_sys/times.h new file mode 100644 index 0000000..bd3cade --- /dev/null +++ b/include/windows_sys/times.h @@ -0,0 +1,38 @@ +#ifndef _TIMES_H +#define _TIMES_H + +#ifdef _WIN32 +#include +#include +#include + +int gettimeofday(struct timeval* t,void* timezone); + +// from linux's sys/times.h + +//#include + +#define __need_clock_t +#include + + +/* Structure describing CPU time used by a process and its children. */ +struct tms + { + clock_t tms_utime; /* User CPU time. */ + clock_t tms_stime; /* System CPU time. */ + + clock_t tms_cutime; /* User CPU time of dead children. */ + clock_t tms_cstime; /* System CPU time of dead children. */ + }; + +/* Store the CPU time used by this process and all its + dead children (and their dead children) in BUFFER. + Return the elapsed real time, or (clock_t) -1 for errors. + All times are in CLK_TCKths of a second. */ +clock_t times (struct tms *__buffer); + +typedef long long suseconds_t ; + +#endif +#endif \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 538cf4d..ee84461 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,9 @@ # Find and add the .cxx files to SRC_FILES -FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx ) +FILE( GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx) + +IF(WIN32) + list(APPEND SRC_FILES ${FDPAPI_INCLUDE_DIRS}/windows_sys/time.cpp ) +ENDIF() # Add the (Static) Project Library using SRC_FILES ADD_LIBRARY( ${FDPAPI} STATIC ${SRC_FILES} ) @@ -16,7 +20,6 @@ TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${jsoncpp_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${yamlcpp_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${CURL_INCLUDE_DIRS} ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${digestpp_SOURCE_DIR} ) -TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${spdlog_SOURCE_DIR}/include ) TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${ghc_SOURCE_DIR}/include ) # Toml 11 is a header only library no need to link TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${tomlcpp_SOURCE_DIR} ) @@ -25,7 +28,6 @@ TARGET_INCLUDE_DIRECTORIES( ${FDPAPI} PUBLIC ${tomlcpp_SOURCE_DIR} ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC jsoncpp_static ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC digestpp ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC yaml-cpp ) -TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC spdlog ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC ghc_filesystem ) TARGET_LINK_LIBRARIES( ${FDPAPI} PUBLIC ${CURL_LIBRARIES} ) diff --git a/src/fdp.cxx b/src/fdp.cxx index 6dadaa1..b383acd 100644 --- a/src/fdp.cxx +++ b/src/fdp.cxx @@ -4,7 +4,6 @@ #include #include -#include "spdlog/spdlog.h" #include "json/json.h" #include "fdp/objects/config.hxx" @@ -13,18 +12,18 @@ #include "fdp/registry/data_io.hxx" #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace FairDataPipeline { /*! ************************************************************************** * @class DataPipelineImpl_ * @brief private pointer-to-implementation class containing all backend methods * * This class performs all behind the scenes operations acting as the backend to - * the user called FDP::DataPipeline class. + * the user called FairDataPipeline::DataPipeline class. * This structure has been chosen to allow for unit tests to be created to test * all methods including private ones. * * @warning The class should not be used directly during implementation, - * but rather via an FDP::DataPipeline instance. + * but rather via an FairDataPipeline::DataPipeline instance. * *****************************************************************************/ class DataPipeline::impl { @@ -35,8 +34,14 @@ namespace FDP { ghc::filesystem::path config_file_path_() const {return config_->get_config_file_path();} ghc::filesystem::path script_file_path_() const {return config_->get_script_file_path();} std::string token_() const {return config_->get_token();} - RESTAPI api_location_() {return config_->get_rest_api_location();} + RESTAPI api_location_() const {return config_->get_rest_api_location();} + impl(const ghc::filesystem::path &config_file_path, + const ghc::filesystem::path &file_system_path, + const std::string& token, + RESTAPI api_location = RESTAPI::LOCAL); + + impl(const impl &dp) = delete; impl& operator=(const impl& ) = delete; public: @@ -50,13 +55,12 @@ namespace FDP { * @param log_level level for the output logging statements * @param api_location whether to use local/remote RestAPI endpoint ***************************************************************************/ - impl(const ghc::filesystem::path &config_file_path, + static sptr construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &file_system_path, const std::string& token, - spdlog::level::level_enum log_level = spdlog::level::info, RESTAPI api_location = RESTAPI::LOCAL); - //impl(const impl &dp) delete; + /** * @brief Default Destructor @@ -65,9 +69,6 @@ namespace FDP { ~impl() = default; - //DataPipelineImpl_& operator=(DataPipelineImpl_ dp); - - /** * @brief Return a path to a given data product * Whilst recording it's meta data for the code run @@ -103,90 +104,83 @@ namespace FDP { std::string get_code_run_uuid() const; }; -#if 0 -DataPipeline::impl::impl(const impl &dp) +DataPipeline::impl::sptr DataPipeline::impl::construct(const ghc::filesystem::path &config_file_path, + const ghc::filesystem::path &script_file_path, + const std::string& token, + RESTAPI api_location) { - impl( dp.config_file_path_() - , dp.script_file_path_() - , token_(), spdlog::level::info - , api_location_() - ); + + sptr pobj = impl::sptr( new impl( config_file_path, + script_file_path, + token, + api_location ) ); + return pobj; } -#endif DataPipeline::impl::impl(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string& token, - spdlog::level::level_enum log_level, RESTAPI api_location) { this->config_ = Config::construct(config_file_path, script_file_path, token, api_location); const std::string api_root_ = config_->get_api_url(); - spdlog::set_default_logger(APILogger); - APILogger->set_level(log_level); + auto the_logger = logger::get_logger(); - APILogger->info("\n[Configuration]\n\t- Config Path: {0}\n\t- API Root: " - "{1}\n\t- FDP API Token: {2}", - config_file_path.string(), api_root_, - token); + the_logger->info() << "\n[Configuration]\n\t- Config Path:" << config_file_path.string() + << "\n\t- API Root: " + << api_root_ + << "\n\t- FDP API Token: " << token; } -ghc::filesystem::path FDP::DataPipeline::impl::link_read(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_read(std::string &data_product){ return config_->link_read(data_product); } -ghc::filesystem::path FDP::DataPipeline::impl::link_write(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::impl::link_write(std::string &data_product){ return config_->link_write(data_product); } -void FDP::DataPipeline::impl::finalise(){ +void FairDataPipeline::DataPipeline::impl::finalise(){ config_->finalise(); } -std::string FDP::DataPipeline::impl::get_code_run_uuid() const { +std::string FairDataPipeline::DataPipeline::impl::get_code_run_uuid() const { return config_->get_code_run_uuid(); } - - - - DataPipeline::~DataPipeline() = default; -DataPipeline::DataPipeline(DataPipeline &&) noexcept = default; -DataPipeline& DataPipeline::operator=(DataPipeline &&) noexcept = default; - -DataPipeline::DataPipeline(const DataPipeline& rhs) - : pimpl_(std::make_shared< DataPipeline::impl >(*rhs.pimpl_)) -{} -DataPipeline::DataPipeline( +DataPipeline::sptr DataPipeline::construct( const std::string &config_file_path, const std::string &script_file_path, - std::string token, - spdlog::level::level_enum log_level ) -: pimpl_(std::make_shared< DataPipeline::impl >(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token, - log_level)) + std::string token ) { - APILogger->debug("DataPipeline: Initialising session '{0}'", pimpl_->get_code_run_uuid()); + return DataPipeline::sptr( new DataPipeline( + config_file_path, + script_file_path, + token ) ); } -DataPipeline& DataPipeline::operator=(const DataPipeline& rhs) { - if (this != &rhs) - pimpl_.reset(new impl(*rhs.pimpl_)); - - return *this; +DataPipeline::DataPipeline( + const std::string &config_file_path, + const std::string &script_file_path, + std::string token ) +: pimpl_( DataPipeline::impl::construct(ghc::filesystem::path(config_file_path), ghc::filesystem::path(script_file_path), token )) +{ + logger::get_logger()->debug() << "DataPipeline: Initialising session '" + << pimpl_->get_code_run_uuid() << "'"; } -ghc::filesystem::path FDP::DataPipeline::link_read(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::link_read(std::string &data_product){ return pimpl_->link_read(data_product); } -ghc::filesystem::path FDP::DataPipeline::link_write(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::DataPipeline::link_write(std::string &data_product){ return pimpl_->link_write(data_product); } -void FDP::DataPipeline::finalise(){ +void FairDataPipeline::DataPipeline::finalise(){ pimpl_->finalise(); } @@ -194,4 +188,4 @@ void FDP::DataPipeline::finalise(){ -}; // namespace FDP +}; // namespace FairDataPipeline diff --git a/src/objects/api_object.cxx b/src/objects/api_object.cxx index a8b459a..e488a23 100644 --- a/src/objects/api_object.cxx +++ b/src/objects/api_object.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/api_object.hxx" -namespace FDP { +namespace FairDataPipeline { ApiObject::ApiObject() : obj_(Json::Value() ) { @@ -16,7 +16,7 @@ namespace FDP { ApiObject::sptr ApiObject::construct(void) { - ApiObject::sptr pobj = std::shared_ptr< ApiObject >( new ApiObject() ); + ApiObject::sptr pobj = ApiObject::sptr( new ApiObject() ); return pobj; } @@ -62,7 +62,7 @@ namespace FDP { * * @return int of the object id */ - int ApiObject::get_id(){ + int ApiObject::get_id() const { return get_id_from_string(get_uri()); } @@ -78,7 +78,7 @@ namespace FDP { * * @return std::string The object type */ - std::string ApiObject::get_type(){ + std::string ApiObject::get_type() const { std::string uri_ = get_uri(); std::size_t id_pos = uri_.find_last_of("/"); std::string uri_prefix = uri_.substr(0, id_pos); diff --git a/src/objects/config.cxx b/src/objects/config.cxx index c745ded..7f64839 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -1,13 +1,13 @@ #include "fdp/objects/config.hxx" -namespace FDP { +namespace FairDataPipeline { Config::sptr Config::construct(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string &token, RESTAPI api_location) { - Config::sptr pobj = std::shared_ptr< Config >( new Config( + Config::sptr pobj = Config::sptr( new Config( config_file_path , script_file_path , token @@ -17,7 +17,7 @@ namespace FDP { -FDP::Config::Config(const ghc::filesystem::path &config_file_path, +FairDataPipeline::Config::Config(const ghc::filesystem::path &config_file_path, const ghc::filesystem::path &script_file_path, const std::string &token, RESTAPI api_location) @@ -25,134 +25,147 @@ FDP::Config::Config(const ghc::filesystem::path &config_file_path, rest_api_location_(api_location) { validate_config(config_file_path, api_location); initialise(api_location); -} + + } + Config::~Config() { - // delete api; } -YAML::Node FDP::Config::parse_yaml(ghc::filesystem::path yaml_path) { - APILogger->debug("[Config]: Reading configuration file '{0}'", - yaml_path.string().c_str()); +YAML::Node FairDataPipeline::Config::parse_yaml(ghc::filesystem::path yaml_path) { + logger::get_logger()->debug() + << "[Config]: Reading configuration file '" << yaml_path.string().c_str() << "'"; return YAML::LoadFile(yaml_path.string().c_str()); } -YAML::Node FDP::Config::meta_data_() const { +YAML::Node FairDataPipeline::Config::meta_data_() const { return config_data_["run_metadata"]; } -YAML::Node FDP::Config::config_writes_() const{ +YAML::Node FairDataPipeline::Config::config_writes_() const{ return config_data_["write"]; } -YAML::Node FDP::Config::config_reads_() const{ +YAML::Node FairDataPipeline::Config::config_reads_() const{ return config_data_["read"]; } -bool FDP::Config::config_has_writes() const{ +bool FairDataPipeline::Config::config_has_writes() const{ if(config_data_["write"]){ return true; } return false; } -bool FDP::Config::config_has_reads() const{ +bool FairDataPipeline::Config::config_has_reads() const{ if(config_data_["read"]){ return true; } return false; } -bool FDP::Config::has_writes() const{ +bool FairDataPipeline::Config::has_writes() const{ return ! writes_.empty(); } -bool FDP::Config::has_reads() const{ +bool FairDataPipeline::Config::has_reads() const{ return ! reads_.empty(); } -bool FDP::Config::has_inputs() const{ +bool FairDataPipeline::Config::has_inputs() const{ return ! inputs_.empty(); } -bool FDP::Config::has_outputs() const{ +bool FairDataPipeline::Config::has_outputs() const{ return ! outputs_.empty(); } -ghc::filesystem::path FDP::Config::get_data_store() const { +ghc::filesystem::path FairDataPipeline::Config::get_data_store() const { return ghc::filesystem::path( - FDP::Config::meta_data_()["write_data_store"].as()); + FairDataPipeline::Config::meta_data_()["write_data_store"].as()); } -std::string FDP::Config::get_default_input_namespace() const { +std::string FairDataPipeline::Config::get_default_input_namespace() const { return meta_data_()["default_input_namespace"].as(); } -std::string FDP::Config::get_default_output_namespace() const { +std::string FairDataPipeline::Config::get_default_output_namespace() const { return meta_data_()["default_output_namespace"].as(); } -void FDP::Config::validate_config(ghc::filesystem::path yaml_path, +void FairDataPipeline::Config::validate_config(ghc::filesystem::path yaml_path, RESTAPI api_location) { config_data_ = parse_yaml(yaml_path); if (!config_data_["run_metadata"]) { - APILogger->error( - "Failed to obtain run metadata, the key 'run_metadata' is not" - " present in the configuration"); + logger::get_logger()->error() + << "Failed to obtain run metadata, the key 'run_metadata' is not" + " present in the configuration"; throw config_parsing_error("No run metadata provided"); } if (api_location == RESTAPI::LOCAL && !meta_data_()["local_data_registry_url"]) { - APILogger->error("Could not determine URL for local repository, key " - "'local_data_registry_url' " - "is absent from configuration"); + logger::get_logger()->error() + << "Could not determine URL for local repository, key " + "'local_data_registry_url' " + "is absent from configuration"; throw config_parsing_error("Failed to read local repository url"); } else if (api_location == RESTAPI::REMOTE && !meta_data_()["remote_data_registry_url"]) { - APILogger->error("Could not determine URL for remote repository, key " - "'remote_data_registry_url' " - "is absent from configuration"); + logger::get_logger()->error() + <<"Could not determine URL for remote repository, key " + "'remote_data_registry_url' " + "is absent from configuration"; throw config_parsing_error("Failed to read remote repository url"); } else if (!(api_location == RESTAPI::LOCAL || api_location == RESTAPI::REMOTE)) { - APILogger->error("Unrecognised API location"); + logger::get_logger()->error() << "Unrecognised API location"; throw config_parsing_error("Failed to resolve registry location"); } if (!meta_data_()["write_data_store"]) { - APILogger->error("Failed to Read: [\"write_data_store\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"write_data_store\"] from " + << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"write_data_store\"] from " + yaml_path.string()); } if (!meta_data_()["default_input_namespace"]) { - APILogger->error("Failed to Read: [\"default_input_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"default_input_namespace\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"default_input_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["default_output_namespace"]) { - APILogger->error("Failed to Read: [\"default_output_namespace\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"default_output_namespace\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"default_output_namespace\"] from " + yaml_path.string()); } if (!meta_data_()["latest_commit"]) { - APILogger->error("Failed to Read: [\"latest_commit\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"latest_commit\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"latest_commit\"] from " + yaml_path.string()); } if (!meta_data_()["remote_repo"]) { - APILogger->error("Failed to Read: [\"remote_repo\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"remote_repo\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"remote_repo\"] from " + yaml_path.string()); } if (!meta_data_()["description"]) { - APILogger->error("Failed to Read: [\"description\"] from {0}", yaml_path.string()); + logger::get_logger()->error() + << "Failed to Read: [\"description\"] from " << yaml_path.string(); throw config_parsing_error("Failed to Read: [\"description\"] from " + yaml_path.string()); } if (!file_exists(script_file_path_.string())) { - APILogger->error("Submission script: {0} does not exist", script_file_path_.string()); + logger::get_logger()->error() + << "Submission script: " + << script_file_path_.string() + << " does not exist"; throw std::runtime_error("Submission script: " + script_file_path_.string() + " does not exist"); } @@ -173,7 +186,7 @@ void FDP::Config::validate_config(ghc::filesystem::path yaml_path, } -void FDP::Config::initialise(RESTAPI api_location) { +void FairDataPipeline::Config::initialise(RESTAPI api_location) { // Set API URL if (api_location == RESTAPI::REMOTE) { api_url_ = API::append_with_forward_slash( @@ -183,10 +196,11 @@ void FDP::Config::initialise(RESTAPI api_location) { meta_data_()["local_data_registry_url"].as()); } - APILogger->info("Reading {0} from local filestore", - config_file_path_.string()); + logger::get_logger()->info() << "Reading " + << config_file_path_.string() + << " from local filestore"; // Create and API object as a shared pointer - api_ = std::make_shared(api_url_); + api_ = API::construct(api_url_); // Get the admin user from registry Json::Value user_json_; @@ -196,7 +210,7 @@ void FDP::Config::initialise(RESTAPI api_location) { this->user_ = ApiObject::from_json( j [0]); if (user_->is_empty()) { - APILogger->error("User: Admin Not Found"); + logger::get_logger()->error() << "User: Admin Not Found"; throw std::runtime_error("User: Admin Not Found"); } @@ -210,8 +224,8 @@ void FDP::Config::initialise(RESTAPI api_location) { this->author_ = ApiObject::from_json( j_author ); if (author_->is_empty()) { - APILogger->error( - "Author for User Admin not found please ensure you have run fair init"); + logger::get_logger()->error() + << "Author for User Admin not found please ensure you have run fair init"; throw std::runtime_error( "Author Not Found: Please ensure you have run fair init"); } @@ -257,7 +271,10 @@ void FDP::Config::initialise(RESTAPI api_location) { config_value_["file_type"] = config_file_type_url_; - APILogger->info("Writing config file {0} to registry", config_file_path_.string()); + logger::get_logger()->info() + << "Writing config file " + << config_file_path_.string() + << " to registry", config_file_path_.string(); Json::Value j_config_obj = api_->post("object", config_value_, token_); @@ -295,7 +312,10 @@ void FDP::Config::initialise(RESTAPI api_location) { script_value_["filetype"] = script_file_type_url_.asString(); script_value_["storage_location"] = script_storage_location_->get_uri(); - APILogger->info("Writing script file {0} to registry", script_file_path_.string()); + logger::get_logger()->info() + << "Writing script file " + << script_file_path_.string() + << " to registry"; Json::Value j_script_obj = api_->post("object", script_value_, token_); this->script_obj_ = ApiObject::from_json( j_script_obj ); @@ -335,12 +355,15 @@ void FDP::Config::initialise(RESTAPI api_location) { code_run_value_["input_urls"] = Json::arrayValue; code_run_value_["output_urls"] = Json::arrayValue; - APILogger->info("Writing new code run to registry"); + logger::get_logger()->info() << "Writing new code run to registry"; Json::Value j_code_run = api_->post("code_run", code_run_value_, token_); this->code_run_ = ApiObject::from_json( j_code_run ); - APILogger->info("Code run {0} successfully generated", code_run_->get_value_as_string("uuid")); + logger::get_logger()->info() + << "Code run " + << code_run_->get_value_as_string("uuid") + << " successfully generated"; } std::string Config::get_config_directory() const{ @@ -352,13 +375,15 @@ std::string Config::get_code_run_uuid() const{ }; -ghc::filesystem::path Config::link_write(std::string &data_product){ +ghc::filesystem::path Config::link_write( const std::string& data_product){ if (!config_has_writes()){ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } - //APILogger->info("Node Type: {0}", config_writes_().Type()); + //logger::get_logger()->info() << "Node Type: " + //<< config_writes_().Type()); YAML::Node currentWrite; @@ -374,30 +399,44 @@ ghc::filesystem::path Config::link_write(std::string &data_product){ } } else{ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentWrite) { - APILogger->error("Config Error: Cannot Find {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find " + << data_product + << " in writes"; + throw config_parsing_error("Config Error: cannot find " + data_product + "in writes"); } if(!currentWrite["description"]) { - APILogger->error("Config Error: Cannot Find description of {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find description of " + << data_product + << " in writes"; throw config_parsing_error("Config Error: cannot find description of " + data_product + "in writes"); } if(!currentWrite["file_type"]) { - APILogger->error("Config Error: Cannot Find file_type of {0} in writes", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find file_type of " + << data_product + << " in writes"; throw config_parsing_error("Config Error: cannot find file_type of " + data_product + "in writes"); } if(!currentWrite["use"]["version"]){ - APILogger->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info() + << "Use: Version not found in " + << data_product + << ", using version 0.0.1 by default"; currentWrite["use"]["version"] = "0.0.1"; } @@ -412,7 +451,7 @@ ghc::filesystem::path Config::link_write(std::string &data_product){ std::string filename_("dat-" + generate_random_hash() + "." + currentWrite["file_type"].as()); ghc::filesystem::path path_ = ghc::filesystem::path(meta_data_()["write_data_store"].as()) / currentWrite["use"]["namespace"].as() / currentWrite["use"]["data_product"].as() / filename_; - APILogger->info("Link Path: {0}", path_.string()); + logger::get_logger()->info() << "Link Path: " << path_.string(); // Create Directory ghc::filesystem::create_directories(path_.parent_path().string()); @@ -429,7 +468,7 @@ ghc::filesystem::path Config::link_write(std::string &data_product){ } -ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ +ghc::filesystem::path FairDataPipeline::Config::link_read( const std::string &data_product){ YAML::Node currentRead; auto it = inputs_.find("data_product"); @@ -449,18 +488,25 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ } } else{ - APILogger->error("Config Error: Write has not been specified in the given config file"); + logger::get_logger()->error() + << "Config Error: Write has not been specified in the given config file"; throw config_parsing_error("Config Error: Write has not been specified in the given config file"); } if(!currentRead) { - APILogger->error("Config Error: Cannot Find {0} in reads", data_product); + logger::get_logger()->error() + << "Config Error: Cannot Find " + << data_product + << " in reads"; throw config_parsing_error("Config Error: cannot find " + data_product + "in reads"); } if(!currentRead["use"]["version"]){ - APILogger->info("Use: Version not found in {0}, using version 0.0.1 by default", data_product); + logger::get_logger()->info() + << "Use: Version not found in " + << data_product + << ", using version 0.0.1 by default"; currentRead["use"]["version"] = "0.0.1"; } @@ -479,7 +525,10 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ ApiObject::sptr namespaceObj = ApiObject::from_json( j_namespace ); if (namespaceObj->is_empty()){ - APILogger->error("Namespace Error: could not find namespace {0} in registry", currentRead["use"]["namespace"].as()); + logger::get_logger()->error() + << "Namespace Error: could not find namespace " + << currentRead["use"]["namespace"].as() + << " in registry"; throw std::runtime_error("Namespace Error: could not find namespace " + currentRead["use"]["namespace"].as() + " in Registry"); } @@ -492,7 +541,10 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ ApiObject::sptr dataProductObj = ApiObject::from_json( j_data_prod_obj ); //ApiObject dataProductObj = ApiObject(api_->get_by_json_query("data_product", dataProductData)[0]); if (dataProductObj->is_empty()){ - APILogger->error("data_product Error: could not find data_product {0} in registry", currentRead["use"]["data_product"].as()); + logger::get_logger()->error() + << "data_product Error: could not find data_product " + << currentRead["use"]["data_product"].as() + << " in registry"; throw std::runtime_error("Namespace Error: could not find data_product " + currentRead["use"]["data_product"].as() + " in Registry"); } @@ -500,7 +552,10 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ ApiObject::sptr obj = ApiObject::from_json( _j_ ); if (obj->is_empty()){ - APILogger->error("data_product Error: could not find data_product object {0} in registry", dataProductObj->get_value_as_string("object")); + logger::get_logger()->error() + << "data_product Error: could not find data_product object " + << dataProductObj->get_value_as_string("object") + << " in registry"; throw std::runtime_error("Namespace Error: could not find data_product object " + dataProductObj->get_value_as_string("object") + " in Registry"); } @@ -510,7 +565,10 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ ApiObject::sptr componentObj = ApiObject::from_json( j_obj_component ); if (componentObj->is_empty()){ - APILogger->error("data_product object Error: could not find data_product component_object for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error() + << "data_product object Error: could not find data_product component_object for " + << obj->get_value_as_string("object") + << " in registry"; throw std::runtime_error("data_product object Error: could not find data_product component_object for " + obj->get_value_as_string("object") + " in Registry"); } @@ -520,7 +578,9 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ ApiObject::sptr storageLocationObj = ApiObject::from_json( j_tmp2 ); if (storageLocationObj->is_empty()){ - APILogger->error("data_product object Error: could not find storage_location for {0} in registry", obj->get_value_as_string("object")); + logger::get_logger()->error() << "data_product object Error: could not find storage_location for " + << obj->get_value_as_string("object") << " in registry"; + throw std::runtime_error("data_product object Error: could not find storage_location for " + obj->get_value_as_string("object") + " in Registry"); } @@ -529,7 +589,10 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ if (storageRootObj->is_empty()){ std::string obj_str = obj->get_value_as_string("object"); - APILogger->error("data_product object Error: could not find storage_root for {0} in registry", obj_str); + + logger::get_logger()->error() + << "data_product object Error: could not find storage_root for " << obj_str << " in registry"; + throw std::runtime_error("data_product object Error: could not find storage_root for " + obj_str + " in Registry"); } @@ -548,15 +611,17 @@ ghc::filesystem::path FDP::Config::link_read(std::string &data_product){ return path_; } -void FDP::Config::finalise(){ +void FairDataPipeline::Config::finalise(){ if(has_writes()){ - std::map::iterator it; + Config::map_type::iterator it; for (it = writes_.begin(); it != writes_.end(); it++){ - IOObject currentWrite = it->second; + IOObject& currentWrite = it->second; if(! file_exists(currentWrite.get_path().string())){ - APILogger->error("File Error: Cannot Find file for write", currentWrite.get_use_data_product()); + logger::get_logger()->error() + << "File Error: Cannot Find file for write" << currentWrite.get_use_data_product(); + throw std::runtime_error("File Error Cannot Find file for write: " + currentWrite.get_use_data_product()); } @@ -667,9 +732,9 @@ void FDP::Config::finalise(){ } if(has_reads()){ - std::map::iterator it; + map_type::iterator it; for (it = reads_.begin(); it != reads_.end(); it++){ - IOObject currentRead = it->second; + IOObject& currentRead = it->second; inputs_[currentRead.get_data_product()] = currentRead; } } @@ -677,31 +742,38 @@ void FDP::Config::finalise(){ Json::Value patch_data; if(has_outputs()){ - std::map::iterator it; + map_type::iterator it; for (it = outputs_.begin(); it != outputs_.end(); it++){ - IOObject currentOutput = it->second; + IOObject& currentOutput = it->second; Json::Value output = currentOutput.get_component_object()->get_uri(); patch_data["outputs"].append(output); - APILogger->info("Writing {} to local registry", currentOutput.get_use_data_product()); + //logger::get_logger()->info() + //<< "Writing + //<< currentOutput.get_use_data_product() + //<< " to local registry"; + logger::get_logger()->info() + << "Writing " << currentOutput.get_use_data_product() << " to local registry"; + } } if(has_inputs()){ - std::map::iterator it; + map_type::iterator it; for (it = inputs_.begin(); it != inputs_.end(); it++){ - IOObject currentInput = it->second; + IOObject& currentInput = it->second; Json::Value input = currentInput.get_component_object()->get_uri(); patch_data["inputs"].append(input); - APILogger->info("Writing {} to local registry", currentInput.get_use_data_product()); + logger::get_logger()->info() + << "Writing " << currentInput.get_use_data_product() << " to local registry"; } } std::string code_run_endpoint = "code_run/" + std::to_string(code_run_->get_id()); - APILogger->info("Code Run: {0}", code_run_endpoint); + logger::get_logger()->info() << "Code Run: " << code_run_endpoint; Json::Value j_code_run = api_->patch(code_run_endpoint, patch_data, token_); this-> code_run_ = ApiObject::from_json( j_code_run ); } -}; // namespace FDP +}; // namespace FairDataPipeline diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 236113b..05ce8ed 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -1,6 +1,6 @@ #include "fdp/objects/metadata.hxx" -namespace FDP { +namespace FairDataPipeline { std::string calculate_hash_from_file(const ghc::filesystem::path &file_path) { if (!ghc::filesystem::exists(file_path)) { throw std::invalid_argument("File '" + file_path.string() + "' not found"); @@ -85,4 +85,4 @@ std::string read_token(const ghc::filesystem::path &token_path){ return key_str_; } -}; // namespace FDP \ No newline at end of file +}; // namespace FairDataPipeline \ No newline at end of file diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 3d4904a..b8e6eb1 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -1,17 +1,24 @@ #include "fdp/registry/api.hxx" -namespace FDP { -static size_t write_str_(void *ptr, size_t size, size_t nmemb, std::string *data) { - data->append((char *)ptr, size * nmemb); - return size * nmemb; +namespace FairDataPipeline { +static size_t write_str_(char *ptr, size_t size, size_t nmemb, void* userdata ) { + std::string* data = static_cast< std::string* >( userdata ); + data->append((char *)ptr, size * nmemb); + return size * nmemb; } -static size_t write_file_(void *ptr, size_t size, size_t nmemb, FILE *stream) { - size_t written_n_ = fwrite(ptr, size, nmemb, stream); - return written_n_; +static size_t write_file_(char*ptr, size_t size, size_t nmemb, void* userdata ) { + FILE* stream = static_cast< FILE* >( userdata ); + size_t written_n_ = fwrite(ptr, size, nmemb, stream); + return written_n_; } -std::string url_encode(std::string url) { +API::sptr API::construct( const std::string& url_root ) +{ + return API::sptr( new API( url_root ) ); +} + +std::string url_encode( const std::string& url) { CURL *curl_ = curl_easy_init(); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); return curl_easy_escape(curl_, url.c_str(), 0); @@ -24,14 +31,18 @@ CURL *API::setup_json_session_(std::string &addr_path, std::string *response, if (!token.empty()) { - APILogger->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug() + << "Adding token: " + << token + << " to headers"; struct curl_slist *headers = NULL; headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, headers); } - APILogger->debug("API:JSONSession: Attempting to access: " + addr_path); + logger::get_logger()->debug() + << "API:JSONSession: Attempting to access: " << addr_path; curl_easy_setopt(curl_, CURLOPT_URL, addr_path.c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_str_); @@ -53,8 +64,10 @@ void API::download_file(const ghc::filesystem::path &url, CURL *curl_ = curl_easy_init(); FILE *file_ = fopen(out_path.string().c_str(), "wb"); - APILogger->debug("API: Downloading file '{0}' -> '{1}'", url.string(), - out_path.string()); + logger::get_logger()->debug() + << "API: Downloading file '" + << url.string() + << "' -> '" << out_path.string() << "'", url.string(); curl_easy_setopt(curl_, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); curl_easy_setopt(curl_, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_file_); @@ -66,8 +79,10 @@ void API::download_file(const ghc::filesystem::path &url, CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, FILE *file) { CURL *curl_ = curl_easy_init(); - APILogger->debug("API:DownloadSession: Attempting to access: " + - addr_path.string()); + + logger::get_logger()->debug() + << "API:DownloadSession: Attempting to access: " << addr_path.string(); + curl_easy_setopt(curl_, CURLOPT_URL, addr_path.string().c_str()); curl_easy_setopt(curl_, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_file_); @@ -102,8 +117,10 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons JSONCPP_STRING err; if (http_code == 0) { - APILogger->error("API:Request: Request to '{0}' returned no response", - search_str_); + logger::get_logger()->error() + << "API:Request: Request to '" + << search_str_ + << "' returned no response"; throw rest_apiquery_error("No response was given"); } @@ -117,9 +134,11 @@ Json::Value API::get_request(const std::string &addr_path, long expected_respons if (!json_reader_->parse(response_str_.c_str(), response_str_.c_str() + response_str_len_, &root_, &err)) { - APILogger->error("API:Query: Response string '{0}' is not JSON parsable. " - "Return Code was {1}", - response_str_, http_code); + logger::get_logger()->error() + << "API:Query: Response string '" + << response_str_ + << "' is not JSON parsable. Return Code was " + << http_code; throw rest_apiquery_error( "Failed to retrieve information from JSON response string"); } @@ -209,7 +228,7 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat const std::string url_path_ = url_root_ + API::append_with_forward_slash(addr_path); const std::string data_ = json_to_string(post_data); - APILogger->debug("API:Post: Post Data\n{0}", data_); + logger::get_logger()->debug() << "API:Post: Post Data\n" << data_; long return_code_; std::string response_; CURL *curl_ = curl_easy_init(); @@ -218,7 +237,10 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat headers = curl_slist_append(headers, "Content-Type: application/json"); if (!token.empty()) { - APILogger->debug("Adding token: {0} to headers", token); + logger::get_logger()->debug() + << "Adding token: " + << token + << " to headers"; headers = curl_slist_append( headers, (std::string("Authorization: token ") + token).c_str()); } @@ -241,7 +263,10 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat } else { curl_easy_cleanup(curl_); curl_global_cleanup(); - APILogger->error("API:Post: Post to '{0}' returned no response", url_path_); + logger::get_logger()->error() + << "API:Post: Post to '" + <info("API:Post Entry Exists attempting to return entry"); + logger::get_logger()->info() + << "API:Post Entry Exists attempting to return entry"; return get_request(API::append_with_forward_slash(addr_path) + json_to_query_string(post_data))[0]; } @@ -276,9 +302,11 @@ Json::Value API::post_patch_request(std::string addr_path, Json::Value &post_dat if (!json_reader_->parse(response_.c_str(), response_.c_str() + response_str_len_, &root_, &err)) { - APILogger->error("API:Post: Response string '{0}' is not JSON parsable. " - "Return Code was {1}", - response_, return_code_); + logger::get_logger()->error() + << "API:Post: Response string '" + << response_ + << "' is not JSON parsable. Return Code was " + << return_code_; throw rest_apiquery_error( "Failed to retrieve information from JSON response string"); } @@ -314,4 +342,4 @@ std::string API::remove_leading_forward_slash(std::string str) { return str; } -}; // namespace FDP +}; // namespace FairDataPipeline diff --git a/src/registry/data_io.cxx b/src/registry/data_io.cxx index 7a17c51..0382a42 100644 --- a/src/registry/data_io.cxx +++ b/src/registry/data_io.cxx @@ -1,9 +1,11 @@ #include "fdp/registry/data_io.hxx" -#include "fdp/objects/config.hxx" -namespace FDP { + +namespace FairDataPipeline { YAML::Node parse_yaml_(ghc::filesystem::path yaml_path) { - APILogger->debug("LocalFileSystem: Reading configuration file '{0}'", - yaml_path.string().c_str()); + logger::get_logger()->debug() + << "LocalFileSystem: Reading configuration file '" + << yaml_path.string() + << "'"; return YAML::LoadFile(yaml_path.string().c_str()); } @@ -38,4 +40,4 @@ double read_point_estimate_from_toml(const ghc::filesystem::path var_address) { std::string get_first_key_(const toml::value data_table) { return data_table.as_table().begin()->first; } -}; // namespace FDP \ No newline at end of file +}; // namespace FairDataPipeline diff --git a/src/utilities/json.cxx b/src/utilities/json.cxx index 8393d0f..e0b944b 100644 --- a/src/utilities/json.cxx +++ b/src/utilities/json.cxx @@ -1,8 +1,8 @@ #include "fdp/utilities/json.hxx" -namespace FDP { +namespace FairDataPipeline { std::string json_to_string(Json::Value &json_data) { Json::StreamWriterBuilder json_str_builder_; return Json::writeString(json_str_builder_, json_data); } -} // namespace FDP \ No newline at end of file +} // namespace FairDataPipeline \ No newline at end of file diff --git a/src/utilities/logging.cxx b/src/utilities/logging.cxx index 8c775a2..6640dc9 100644 --- a/src/utilities/logging.cxx +++ b/src/utilities/logging.cxx @@ -1,20 +1,221 @@ +#include +#include +#ifdef _WIN32 + #include +#else + #include +#endif +#include +#include + #include "fdp/utilities/logging.hxx" -namespace FDP { +namespace FairDataPipeline { + namespace logging { + + const std::string& to_string( enum LOG_LEVEL lvl ) + { + static const std::string strs[] = {"TRACE","DEBUG","INFO","WARN","ERROR","CRITICAL","OFF"}; + return strs[ lvl ]; + } + + enum LOG_LEVEL to_LOG_LEVEL( const std::string& s ) + { + enum LOG_LEVEL lvl = LOG_LEVEL::OFF; + static std::map< std::string, enum LOG_LEVEL > m = { + {"TRACE", LOG_LEVEL::TRACE } + , {"DEBUG", LOG_LEVEL::DEBUG } + , {"INFO", LOG_LEVEL::INFO } + , {"WARN", LOG_LEVEL::WARN } + , {"ERROR", LOG_LEVEL::ERROR } + , {"CRITICAL", LOG_LEVEL::CRITICAL } + , {"OFF", LOG_LEVEL::OFF } + }; + + auto it = m.find( s ); + + if( it != m.end() ) + lvl = it->second; + + return lvl; + } + + static std::string curtime() + { + timeval curTime; + gettimeofday( &curTime, NULL ); + int milli = curTime.tv_usec / 1000; + char buffer[ 80 ]; + std::time_t _tv_sec = curTime.tv_sec; + strftime( buffer, 80, "%Y-%m-%d %H:%M:%S", localtime( &_tv_sec )); + std::string currentTime( 84, 0 ); + + sprintf( ¤tTime[0], "%s.%03d", buffer, milli ); + return std::move( currentTime ); + } + + Logger::Logger( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) + : _sink(sink), _name(name) + { + _log_lvl = lvl; + } + + Logger::sptr Logger::create( enum LOG_LEVEL lvl, Sink::sptr sink, std::string name ) + { + return sptr( new Logger(lvl, sink, name)); + } + + Logger::MsgBuilder::MsgBuilder( enum LOG_LEVEL msg_lvl, Logger* logger ) + : _msg_lvl( msg_lvl ), _logger(logger) + { + } + + Logger::MsgBuilder::MsgBuilder( const Logger::MsgBuilder& rhs ) //: _sink( rhs._sink ) + { + this->_msg_lvl = rhs. _msg_lvl; + _logger = rhs._logger; + } + + Logger::MsgBuilder::~MsgBuilder() + { + if( _oss.tellp() > 0 ) + { + *this << "\n"; + + std::string msg = _oss.str(); + + this->_logger->sink()->execute( _logger, _msg_lvl, msg ); + } + } + + Logger::MsgBuilder Logger::info() + { + return std::move( MsgBuilder( INFO, this ) ); + } + + Logger::MsgBuilder Logger::debug() + { + return std::move( MsgBuilder( DEBUG, this ) ); + } + + Logger::MsgBuilder Logger::trace() + { + return std::move( MsgBuilder( TRACE, this ) ); + } + + Logger::MsgBuilder Logger::warn() + { + return std::move( MsgBuilder( WARN, this ) ); + } + + Logger:: MsgBuilder Logger::critical() + { + return std::move( MsgBuilder( WARN, this ) ); + } + + Logger::MsgBuilder Logger::error() + { + return std::move( MsgBuilder( ERROR, this ) ); + } + + Sink::Sink( enum LOG_LEVEL log_lvl ) + : ISink(), _log_lvl(log_lvl) + { + _fmtr = SinkFormatter::create(); + } + + enum LOG_LEVEL Sink::log_level(){ return _log_lvl;} + + void Sink::log_level( enum LOG_LEVEL log_lvl ){ _log_lvl = log_lvl;} + + bool Sink::should_log( enum LOG_LEVEL msg_lvl ){ return msg_lvl >= this->log_level();} + + int Sink::execute( Logger* logger, enum LOG_LEVEL msg_lvl, const std::string& msg ) + { + if( this->should_log( msg_lvl ) ) + { + if( NULL != _fmtr ) + { + //std::ostringstream oss; + //oss << _fmtr->header( msg_lvl, logger ); + //oss << msg; + // + //this->log( msg_lvl, oss.str() ); + std::string s = _fmtr->header(msg_lvl, logger); + s += msg; + + this->log( msg_lvl, s ); + } + else + this->log( msg_lvl, msg ); + } + return 0; + } + + OStreamSink::sptr OStreamSink::create( enum LOG_LEVEL lvl, std::ostream& os ) + { + return sptr( new OStreamSink( lvl, os ) ); + } + + int OStreamSink::log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + _os << s; + + return 0; + } - /*! ************************************************************************** - * @brief Create a the global logger pointer - * @author K. Zarebski (UKAEA) - * - * @return a shared pointer to the global logger instance - ****************************************************************************/ - static logger::sptr create_logger() { - if (!spdlog::get("FDPAPI")) { - return spdlog::stdout_color_st("FDPAPI"); - } else { - return spdlog::get("FDPAPI"); + OStreamSink::OStreamSink(enum LOG_LEVEL log_lvl, std::ostream& os) : Sink(log_lvl), _os(os) + { } + + + std::string SinkFormatter::header( enum LOG_LEVEL msg_lvl, Logger* logger ) + { + std::string dt = curtime(); + std::ostringstream oss; + + oss << "[" << dt << "] "; + + if( !(logger->name().empty()) ) + oss << "[" << logger->name() << "] "; + + oss << "[" << to_string( msg_lvl ) << "] "; + + return std::move( oss.str() ); + } + + int CompositeSink::log( enum LOG_LEVEL msg_lvl, const std::string& s) + { + if( this->should_log( msg_lvl ) ) + { + for( int i = 0; i < _sinks.size(); ++i ) + { + Sink::sptr sink = _sinks[i]; + sink->log( msg_lvl, s ); + } + } + + return 0; + } + } + + + logger::logger_sptr logger::_instance = NULL; + + logger::logger_sptr logger::get_logger() + { + if( NULL == logger::_instance ) + { + enum logging::LOG_LEVEL log_lvl = logging::OFF; + + if( const char* env = std::getenv( "FDP_LOG_LEVEL" ) ) + log_lvl = logging::to_LOG_LEVEL( env ); + + auto sink = logging::OStreamSink::create( log_lvl, std::cout ); + logger::_instance = logging::Logger::create( log_lvl, sink, "FDPAPI" ); + } + + return logger::_instance; } - const logger::sptr APILogger = create_logger(); -} // namespace FDP +} // namespace FairDataPipeline diff --git a/src/utilities/semver.cxx b/src/utilities/semver.cxx index c84ee15..4db3ca4 100644 --- a/src/utilities/semver.cxx +++ b/src/utilities/semver.cxx @@ -1,6 +1,6 @@ #include "fdp/utilities/semver.hxx" -namespace FDP { +namespace FairDataPipeline { Versioning::version::version(const std::string version_str) { const std::string delim_ = "."; const std::string meta_delim_ = "+"; @@ -83,4 +83,4 @@ std::string Versioning::version::get_tag_str() const { return ""; }; } -} // namespace FDP \ No newline at end of file +} // namespace FairDataPipeline \ No newline at end of file diff --git a/test/test_api.cxx b/test/test_api.cxx index 8d82491..c49bcfb 100644 --- a/test/test_api.cxx +++ b/test/test_api.cxx @@ -9,7 +9,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace FairDataPipeline; class ApiTest : public ::testing::Test { protected: @@ -26,19 +26,22 @@ class ApiTest : public ::testing::Test { } ghc::filesystem::path read_csv_config = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; std::string api_url_ = "http://127.0.0.1:8000/api"; - FDP::API *api_ = new FDP::API(api_url_); + + FairDataPipeline::API::sptr api_ = FairDataPipeline::API::construct(api_url_); + std::string token = read_token(ghc::filesystem::path(getHomeDirectory()) / ".fair" / "registry" / "token"); void TearDown() override { - delete api_; } }; //! [TestGetById] TEST_F(ApiTest, TestGetById) { Json::Value author = api_->get_request(std::string("author/?name=Interface%20Test"))[0]["url"]; - APILogger->info("Author: {0}", author.asString()); + logger::get_logger()->info() + << "Author: " + << author.asString(); int author_id_ = ApiObject::get_id_from_string( author.asString() ); ASSERT_EQ(api_->get_by_id(std::string("author"), author_id_)["name"].asString(), std::string("Interface Test")); } //! [TestGetById] diff --git a/test/test_api_object.cxx b/test/test_api_object.cxx index d36d2f4..884a317 100644 --- a/test/test_api_object.cxx +++ b/test/test_api_object.cxx @@ -7,7 +7,7 @@ #include "fdp/utilities/json.hxx" #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace FairDataPipeline; class ApiObjectTest : public ::testing::Test { protected: diff --git a/test/test_config.cxx b/test/test_config.cxx index a27e4cb..d12a3fd 100644 --- a/test/test_config.cxx +++ b/test/test_config.cxx @@ -14,12 +14,13 @@ #include "fdp/utilities/logging.hxx" -using namespace FDP; +using namespace FairDataPipeline; class ConfigTest : public ::testing::Test { protected: void SetUp() override {} - std::string getHomeDirectory() { + + static std::string getHomeDirectory() { std::string HomeDirectory; #ifdef _WIN32 HomeDirectory = getenv("HOMEDRIVE"); @@ -30,35 +31,40 @@ class ConfigTest : public ::testing::Test { return HomeDirectory; } - Config::sptr config(bool use_local = true, std::string config = "write_csv.yaml") { + static Config::sptr config(bool use_local = true, std::string config = "write_csv.yaml") { + + std::string home_dir = getHomeDirectory(); + std::string token = + read_token (ghc::filesystem::path( home_dir) / ".fair" / "registry" / "token"); - const ghc::filesystem::path config_path_ = - ghc::filesystem::path(TESTDIR) / "data" / config; - const ghc::filesystem::path script_path_ = - ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); + const ghc::filesystem::path config_path_ = + ghc::filesystem::path(TESTDIR) / "data" / config; + const ghc::filesystem::path script_path_ = + ghc::filesystem::path(TESTDIR) / "test_script.sh"; + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); - return Config::construct(config_path_, script_path_, token, - static_cast(use_local)); + return Config::construct(config_path_, script_path_, token, + use_local ? RESTAPI::LOCAL : RESTAPI::REMOTE ); } - std::string token = - read_token(ghc::filesystem::path(getHomeDirectory()) / - ".fair" / "registry" / "token"); void TearDown() override {} }; TEST_F(ConfigTest, TestMetaData) { - EXPECT_EQ(config()->meta_data_()["description"].as(), "Write csv file"); - EXPECT_EQ(config()->meta_data_()["local_data_registry_url"].as(), "http://127.0.0.1:8000/api/"); - EXPECT_EQ(config()->meta_data_()["remote_data_registry_url"].as(), "https://data.scrc.uk/api/"); - EXPECT_EQ(config()->meta_data_()["default_input_namespace"].as(), "testing"); - EXPECT_EQ(config()->meta_data_()["default_output_namespace"].as(), "testing"); - //EXPECT_EQ(config()->meta_data_()["write_data_store"].as(), "tmp"); - EXPECT_EQ(config()->meta_data_()["local_repo"].as(), "./"); - EXPECT_EQ(config()->meta_data_()["public"].as(), "true"); - EXPECT_EQ(config()->meta_data_()["latest_commit"].as(), "52008720d240693150e96021ea34ac6fffe05870"); - EXPECT_EQ(config()->meta_data_()["remote_repo"].as(), "https://github.com/FAIRDataPipeline/cppDataPipeline"); + + auto cnf = config(); + auto meta = cnf->meta_data_(); + + EXPECT_EQ( meta["description"].as(), "Write csv file"); + EXPECT_EQ( meta["local_data_registry_url"].as(), "http://127.0.0.1:8000/api/"); + EXPECT_EQ( meta["remote_data_registry_url"].as(), "https://data.scrc.uk/api/"); + EXPECT_EQ( meta["default_input_namespace"].as(), "testing"); + EXPECT_EQ( meta["default_output_namespace"].as(), "testing"); + //EXPECT_EQ(cnf->meta_data_()["write_data_store"].as(), "tmp"); + EXPECT_EQ( meta["local_repo"].as(), "./"); + EXPECT_EQ( meta["public"].as(), "true"); + EXPECT_EQ( meta["latest_commit"].as(), "52008720d240693150e96021ea34ac6fffe05870"); + EXPECT_EQ( meta["remote_repo"].as(), "https://github.com/FAIRDataPipeline/cppDataPipeline"); } TEST_F(ConfigTest, TestLinkWrite){ diff --git a/test/test_pimpl.cxx b/test/test_pimpl.cxx index d716d0e..d5d2a60 100644 --- a/test/test_pimpl.cxx +++ b/test/test_pimpl.cxx @@ -12,7 +12,7 @@ #include "gtest/gtest.h" -using namespace FDP; +using namespace FairDataPipeline; class PimplTest : public ::testing::Test { protected: @@ -39,11 +39,11 @@ TEST_F(PimplTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "data" / "write_csv.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); - DataPipeline dp = DataPipeline(config_path_.string(), script_path_.string(), token, spdlog::level::debug); + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); + DataPipeline::sptr dp = DataPipeline::construct(config_path_.string(), script_path_.string(), token ); std::string data_product = "test/csv"; - ghc::filesystem::path currentLink = ghc::filesystem::path(dp.link_write(data_product)); + ghc::filesystem::path currentLink = ghc::filesystem::path(dp->link_write(data_product)); EXPECT_GT(currentLink.string().size(), 1); std::ofstream testCSV; @@ -51,16 +51,16 @@ TEST_F(PimplTest, TestDataPipelineInit) { testCSV << "Test"; testCSV.close(); - dp.finalise(); + dp->finalise(); ghc::filesystem::path config_path_read_ = ghc::filesystem::path(TESTDIR) / "data" / "read_csv.yaml"; - dp = DataPipeline(config_path_read_.string(), script_path_.string(), token, spdlog::level::debug); + dp = DataPipeline::construct(config_path_read_.string(), script_path_.string(), token ); - currentLink = dp.link_read(data_product); + currentLink = dp->link_read(data_product); EXPECT_GT(currentLink.string().size(), 1); - dp.finalise(); + dp->finalise(); } diff --git a/test/test_registry.cxx b/test/test_registry.cxx index 0123a20..c672aee 100644 --- a/test/test_registry.cxx +++ b/test/test_registry.cxx @@ -9,7 +9,7 @@ #include #include -using namespace FDP; +using namespace FairDataPipeline; class RegistryTest : public ::testing::Test { protected: @@ -25,19 +25,19 @@ class RegistryTest : public ::testing::Test { return HomeDirectory; } - DataPipeline *init_pipeline(bool use_local = true) { + DataPipeline::sptr init_pipeline(bool use_local = true) { const ghc::filesystem::path config_path_ = ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); - return new DataPipeline( + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); + + return DataPipeline::construct( config_path_.string(), script_path_.string(), - token, - spdlog::level::debug ); + token ); } std::string token = @@ -51,13 +51,17 @@ TEST_F(RegistryTest, TestDataPipelineInit) { ghc::filesystem::path(TESTDIR) / "config.yaml"; const ghc::filesystem::path script_path_ = ghc::filesystem::path(TESTDIR) / "test_script.sh"; - APILogger->set_level(spdlog::level::debug); - DataPipeline(config_path_.string(), script_path_.string(), token, spdlog::level::debug); + + logger::get_logger()->set_level( logging::LOG_LEVEL::DEBUG ); + + auto dp = DataPipeline::construct( config_path_.string() + , script_path_.string() + , token ); } TEST_F(RegistryTest, TestLogLevelSet) { init_pipeline(); - ASSERT_EQ(spdlog::get_level(), spdlog::level::debug); + ASSERT_EQ( logger::get_logger()->get_level(), logging::LOG_LEVEL::DEBUG ); } TEST_F(RegistryTest, TestURLEncode) { @@ -72,3 +76,15 @@ TEST_F(RegistryTest, TestHashFile) { calculate_hash_from_file(ghc::filesystem::path(TESTDIR) / "config.yaml"); std::cout << "HASH: " << file_hash_ << std::endl; } + +TEST_F(RegistryTest, TestLogger) { + logging::OStreamSink::sptr sink = logging::OStreamSink::create( logging::INFO, std::cout ); + + logging::Logger::sptr logger = logging::Logger::create( logging::INFO, sink ); + logger->trace() << " " << 10 << " " << "TRACE"; + logger->info() << " " << 50 << " " << "INFO"; + + logger->warn() << " " << 100 << " " << "WARN"; + + logger::get_logger()->info() << " WARNINF"; +} diff --git a/test/test_utilities.cxx b/test/test_utilities.cxx index 7247c6e..01a0004 100644 --- a/test/test_utilities.cxx +++ b/test/test_utilities.cxx @@ -9,7 +9,7 @@ #include "json/reader.h" -using namespace FDP; +using namespace FairDataPipeline; TEST(FDPAPITest, TestSemVerComparisons) { Versioning::version v1{1, 4, 5};