From c13dee1e135640cb3dfe0c7de8afab2b34561f20 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 13 Feb 2022 11:33:37 +0000 Subject: [PATCH 1/5] #50 Ad const stpecifier and tests for new environment --- execute_tests.sh | 3 +++ src/gymfcpp/frozen_lake_env.cpp | 2 +- src/gymfcpp/frozen_lake_env.h | 2 +- tests/CMakeLists.txt | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/execute_tests.sh b/execute_tests.sh index abc3bc53..da707173 100755 --- a/execute_tests.sh +++ b/execute_tests.sh @@ -25,3 +25,6 @@ cd .. echo "Running TiledCartPole environment tests" cd test_tiled_cart_pole/ && ./test_tiled_cart_pole cd .. +echo "Running GridWorld environment tests" +cd test_grid_world/ && ./test_grid_world +cd .. diff --git a/src/gymfcpp/frozen_lake_env.cpp b/src/gymfcpp/frozen_lake_env.cpp index e0066552..d3dde617 100644 --- a/src/gymfcpp/frozen_lake_env.cpp +++ b/src/gymfcpp/frozen_lake_env.cpp @@ -41,7 +41,7 @@ discrete_state_space_frozen_lake<8>::sample(){ } template -std::string FrozenLakeData::name = "FrozenLake"; +const std::string FrozenLakeData::name = "FrozenLake"; template diff --git a/src/gymfcpp/frozen_lake_env.h b/src/gymfcpp/frozen_lake_env.h index d4e2172b..ffdd77a3 100644 --- a/src/gymfcpp/frozen_lake_env.h +++ b/src/gymfcpp/frozen_lake_env.h @@ -143,7 +143,7 @@ struct FrozenLakeData /// /// \brief name /// - static std::string name; + static const std::string name; /// /// \brief time_step_t. The type of the time step diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d8a17c79..7b1835f1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,4 +38,5 @@ ADD_SUBDIRECTORY(test_mountain_car) ADD_SUBDIRECTORY(test_taxi) ADD_SUBDIRECTORY(test_tiled_cart_pole) ADD_SUBDIRECTORY(test_numpy_cpp_utils) +ADD_SUBDIRECTORY(test_grid_world) From 57dc40aa9ce92c1093b4115c47b094613fbb813a Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 13 Feb 2022 11:34:21 +0000 Subject: [PATCH 2/5] #50 Add Gridworld environment --- src/gymfcpp/grid_world_env.h | 159 +++++++++++++++++ tests/test_grid_world/CMakeLists.txt | 21 +++ tests/test_grid_world/test_grid_world.cpp | 205 ++++++++++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 src/gymfcpp/grid_world_env.h create mode 100644 tests/test_grid_world/CMakeLists.txt create mode 100644 tests/test_grid_world/test_grid_world.cpp diff --git a/src/gymfcpp/grid_world_env.h b/src/gymfcpp/grid_world_env.h new file mode 100644 index 00000000..aa23707b --- /dev/null +++ b/src/gymfcpp/grid_world_env.h @@ -0,0 +1,159 @@ +#ifndef GRID_WORLD_ENV_H +#define GRID_WORLD_ENV_H + +#include "gymfcpp/gymfcpp_types.h" +#include "gymfcpp/time_step.h" +#include "gymfcpp/discrete_space.h" +#include "gymfcpp/env_mixin.h" +#include "gymfcpp/extern/enum.h" + +/// +/// Different namespace so that we differentiate +/// from OpenAI-Gym environment +/// +namespace rlenvs +{ + +// still we may want to use some utilities +using namespace gymfcpp; + +/** + * Implements the Gridworld environment from the + * book Deep Reinforcement Learning in Action by Manning publications. + * You can find the original environment here: + * https://github.com/DeepReinforcementLearning/DeepReinforcementLearningInAction + * */ + +template +struct GridworldData +{ + /// + /// \brief action_space_t. The type of the action space + /// + typedef DiscreteSpace<4> action_space_type; + + /// + /// \brief action_t + /// + typedef action_space_type::item_t action_type; + + /// + /// \brief state_space_type + /// + typedef discrete_state_space_frozen_lake state_space_type; + + /// + /// \brief state_type + /// + typedef typename state_space_type::item_type state_type; + + /// + /// \brief state_boost_python_t + /// + typedef boost::python::list state_boost_python_type; + + /// + /// \brief name + /// + static const std::string name; + + /// + /// \brief time_step_t. The type of the time step + /// + typedef TimeStep time_step_type; + + /// + /// \brief state_transform_from_boost + /// \param boost_type + /// \return + /// + static state_type state_transform_from_boost(state_boost_python_type /*boost_type*/); + + /// + /// \brief extract_state + /// \param gym_namespace + /// \return + /// + static state_type extract_state(obj_t /*gym_namespace*/, std::string /*result_name*/); + + /// + /// \brief extract_state_from_reset + /// \param gym_namespace + /// \param py_env_n + /// \return + /// + static state_type extract_state_from_reset(obj_t gym_namespace, std::string py_state_name, std::string result_name); + + /// + /// \brief extract_state_from_step + /// \param gym_namespace + /// \param py_state_name + /// \param result_name + /// \return + /// + static state_type extract_state_from_step(obj_t gym_namespace, std::string py_state_name, std::string result_name); + +}; + +template +class Gridworld final: protected EnvMixin> +{ +public: + + /// + /// + /// + Gridworld(std::string version, bool create=true); + + + /// + /// \brief Expose the functionality this class is using + /// from the Mixin + /// + + using EnvMixin>::full_name; + using EnvMixin>::is_created; + using EnvMixin>::version; + using EnvMixin>::idx; + using EnvMixin>::py_env_name; + using EnvMixin>::py_reset_result_name; + using EnvMixin>::py_step_result_name; + using EnvMixin>::py_state_name; + + /// + /// \brief make. Builds the environment. Optionally we can choose if the + /// environment will be slippery + /// + void make(); + + /// + /// \brief n_states. Returns the number of states + /// + uint_t n_states()const noexcept{ return side_size == 4 ? 16 : 64; } + + /// + /// \brief n_actions. Returns the number of actions + /// + uint_t n_actions()const noexcept{return action_space_type::size;} + + /// + /// \brief step + /// \param action + /// \return + /// + time_step_type step(action_type action, bool query_extra=false); + + /// + /// \brief render + /// + void render(); + + void close(); + + void reset() + +}; + +} + +#endif // GRID_WORLD_ENV_H diff --git a/tests/test_grid_world/CMakeLists.txt b/tests/test_grid_world/CMakeLists.txt new file mode 100644 index 00000000..6c34ca43 --- /dev/null +++ b/tests/test_grid_world/CMakeLists.txt @@ -0,0 +1,21 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.6) + +SET(SOURCE test_grid_world.cpp) +SET(EXECUTABLE test_grid_world) + +ADD_EXECUTABLE(${EXECUTABLE} ${SOURCE}) + +TARGET_LINK_LIBRARIES(${EXECUTABLE} gymfcpplib) + +IF( USE_PYTORCH ) +TARGET_LINK_LIBRARIES(${EXECUTABLE} ${TORCH_LIBRARIES}) +ENDIF() + +TARGET_LINK_LIBRARIES(${EXECUTABLE} python3.8) +TARGET_LINK_LIBRARIES(${EXECUTABLE} boost_python38) +TARGET_LINK_LIBRARIES(${EXECUTABLE} boost_system) +TARGET_LINK_LIBRARIES(${EXECUTABLE} boost_numpy38) +TARGET_LINK_LIBRARIES(${EXECUTABLE} gtest) +TARGET_LINK_LIBRARIES(${EXECUTABLE} gtest_main) # so that tests dont need to have a main +TARGET_LINK_LIBRARIES(${EXECUTABLE} pthread) + diff --git a/tests/test_grid_world/test_grid_world.cpp b/tests/test_grid_world/test_grid_world.cpp new file mode 100644 index 00000000..3df507de --- /dev/null +++ b/tests/test_grid_world/test_grid_world.cpp @@ -0,0 +1,205 @@ +#include "gymfcpp/frozen_lake_env.h" +#include "gymfcpp/time_step.h" +#include "gymfcpp/time_step_type.h" +#include "gymfcpp/gymfcpp_types.h" + +#include +#include + +namespace{ + +using gymfcpp::uint_t; +using gymfcpp::real_t; + +} + + +TEST(TestFrozenLake, TestConstructor4x4) { + + try{ + + Py_Initialize(); + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + gymfcpp::FrozenLake<4> env("v0", main_namespace); + + ASSERT_EQ(env.n_states(), static_cast(16)); + ASSERT_EQ(env.n_actions(), static_cast(4)); + ASSERT_EQ(env.map_type(), "4x4"); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL(); + } +} + +TEST(TestFrozenLake, TestConstructor8x8) { + + try{ + + Py_Initialize(); + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + gymfcpp::FrozenLake<8> env("v0", main_namespace); + + ASSERT_EQ(env.n_states(), static_cast(64)); + ASSERT_EQ(env.n_actions(), static_cast(4)); + ASSERT_EQ(env.map_type(), "8x8"); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL(); + } +} + +TEST(TestFrozenLake, Test_Make) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + + //ASSERT_EQ(env.n_states(), static_cast(16)); + //ASSERT_EQ(env.n_actions(), static_cast(4)); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL(); + } +} + + +TEST(TestFrozenLake, Test_Reset) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + + auto state = env.reset(); + ASSERT_TRUE(state.first()); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL()<<"Error could not reset the environment"; + } +} + +TEST(TestFrozenLake, Test_Step) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + + auto step_result = env.step(0); + ASSERT_TRUE(step_result.mid()); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL()<<"Error could not step in the environment"; + } +} + +TEST(TestFrozenLake, Test_Step_With_Query) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + + auto step_result = env.step(0, true); + ASSERT_TRUE(step_result.mid()); + ASSERT_DOUBLE_EQ(step_result.get_extra("prob"), 0.3333333333333333); + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL()<<"Error could not step in the environment"; + } +} + + +TEST(TestFrozenLake, Test_Get_Dynamics) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + + auto dynamics = env.p(1, 3); + + ASSERT_EQ(dynamics.size(), static_cast(3) ); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL()<<"Error could not step in the environment"; + } +} + +TEST(TestFrozenLake, TestRender) +{ + + try{ + + Py_Initialize(); + + auto main_module = boost::python::import("__main__"); + auto main_namespace = main_module.attr("__dict__"); + + gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + env.make(); + env.reset(); + + env.render(gymfcpp::RenderModeType::human); + + } + catch(const boost::python::error_already_set&) + { + PyErr_Print(); + FAIL()<<"Error could not step in the environment"; + } +} + From 237d280dc5f58cbf80305147655be15907084a76 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 16 Feb 2022 20:29:17 +0000 Subject: [PATCH 3/5] #50 Update implementation --- src/gymfcpp/cart_pole_env.cpp | 22 -- src/gymfcpp/grid_world_env.h | 371 ++++++++++++++++++---- tests/test_grid_world/test_grid_world.cpp | 45 ++- 3 files changed, 330 insertions(+), 108 deletions(-) diff --git a/src/gymfcpp/cart_pole_env.cpp b/src/gymfcpp/cart_pole_env.cpp index 804f66ba..2dcf36ed 100644 --- a/src/gymfcpp/cart_pole_env.cpp +++ b/src/gymfcpp/cart_pole_env.cpp @@ -178,28 +178,6 @@ CartPole::make(){ is_created = true; } -/*CartPole::time_step_type -CartPole::reset(){ - -#ifdef GYMFCPP_DEBUG - assert(is_created && "Environment has not been created"); -#endif - - auto cpp_str = std::string(CartPole::py_reset_result_name + " = "); - cpp_str += CartPole::py_env_name + ".reset().tolist()"; - - // reset the python environment - boost::python::exec(cpp_str.c_str(), gym_namespace); - - // the observation - auto observation = boost::python::extract(gym_namespace[CartPole::py_reset_result_name]); - auto obs = extract_obs(observation); - - current_state = time_step_type(TimeStepTp::FIRST, 0.0, obs); - return current_state; - -}*/ - CartPole::time_step_type CartPole::step(const action_type action){ diff --git a/src/gymfcpp/grid_world_env.h b/src/gymfcpp/grid_world_env.h index aa23707b..e93fe068 100644 --- a/src/gymfcpp/grid_world_env.h +++ b/src/gymfcpp/grid_world_env.h @@ -1,11 +1,27 @@ #ifndef GRID_WORLD_ENV_H #define GRID_WORLD_ENV_H +/** + * Implements the Gridworld environment from the + * book Deep Reinforcement Learning in Action by Manning publications. + * You can find the original environment here: + * https://github.com/DeepReinforcementLearning/DeepReinforcementLearningInAction + * */ + #include "gymfcpp/gymfcpp_types.h" #include "gymfcpp/time_step.h" #include "gymfcpp/discrete_space.h" #include "gymfcpp/env_mixin.h" #include "gymfcpp/extern/enum.h" +#include "gymfcpp/gymfcpp_config.h" + +#ifdef GYMFCPP_DEBUG +#include +#endif + +#include +#include +#include /// /// Different namespace so that we differentiate @@ -17,16 +33,44 @@ namespace rlenvs // still we may want to use some utilities using namespace gymfcpp; -/** - * Implements the Gridworld environment from the - * book Deep Reinforcement Learning in Action by Manning publications. - * You can find the original environment here: - * https://github.com/DeepReinforcementLearning/DeepReinforcementLearningInAction - * */ +/// +/// \brief The RenderModeType enum +/// +BETTER_ENUM(GridworldInitType, char, STATIC=0, RANDOM, PLAYER, INVALID_TYPE); + + +/// +/// \brief to_string. Returns the RenderModeType to its stringrepresentation +/// \param type The RenderModeType to convert +/// \return std::string +inline +std::string to_string(GridworldInitType type){return type._to_string();} + + + +/// +/// The Gridworld class models a square board. There are three ways to initialize the board. +/// - static +/// - random +/// - player +/// See the GridworldInitType enumeration. +/// Static initialization means that the objects on the board are initialized at the same predetermined locations. +/// Player initialization means that the player is initialized at a random position on the board. +/// Random initialization means that all the objects are placed randomly +/// template -struct GridworldData +class Gridworld final { +public: + + static_assert (side_size >= 4, "The side size should be greater than 4"); + + /// + /// \brief name + /// + static const std::string name; + /// /// \brief action_space_t. The type of the action space /// @@ -38,122 +82,323 @@ struct GridworldData typedef action_space_type::item_t action_type; /// - /// \brief state_space_type + /// \brief action_space_t. The type of the action space /// - typedef discrete_state_space_frozen_lake state_space_type; + typedef DiscreteSpace state_space_type; /// /// \brief state_type /// - typedef typename state_space_type::item_type state_type; + typedef typename state_space_type::item_t state_type; /// - /// \brief state_boost_python_t + /// \brief time_step_t. The type of the time step /// - typedef boost::python::list state_boost_python_type; + typedef TimeStep time_step_type; /// - /// \brief name + /// \brief Constructor /// - static const std::string name; + Gridworld(std::string version, GridworldInitType init_mode, bool create=true); /// - /// \brief time_step_t. The type of the time step + /// \brief version + /// \return /// - typedef TimeStep time_step_type; + std::string version()const noexcept{return version_;} /// - /// \brief state_transform_from_boost - /// \param boost_type + /// \brief init_type /// \return /// - static state_type state_transform_from_boost(state_boost_python_type /*boost_type*/); + GridworldInitType init_type()const noexcept{return init_mode_;} /// - /// \brief extract_state - /// \param gym_namespace + /// \brief is_created /// \return /// - static state_type extract_state(obj_t /*gym_namespace*/, std::string /*result_name*/); + bool is_created()const noexcept{return is_created_;} /// - /// \brief extract_state_from_reset - /// \param gym_namespace - /// \param py_env_n - /// \return + /// \brief make. Builds the environment. Optionally we can choose if the + /// environment will be slippery + /// + void make(); + + /// + /// \brief n_states. Returns the number of states /// - static state_type extract_state_from_reset(obj_t gym_namespace, std::string py_state_name, std::string result_name); + uint_t n_states()const noexcept{ return side_size * side_size; } /// - /// \brief extract_state_from_step - /// \param gym_namespace - /// \param py_state_name - /// \param result_name + /// \brief n_actions. Returns the number of actions + /// + uint_t n_actions()const noexcept{return action_space_type::size;} + + /// + /// \brief step + /// \param action /// \return /// - static state_type extract_state_from_step(obj_t gym_namespace, std::string py_state_name, std::string result_name); + time_step_type step(action_type action); -}; + /// + /// \brief render + /// + void render(); -template -class Gridworld final: protected EnvMixin> -{ -public: + /// + /// \brief close + /// + void close(); /// /// /// - Gridworld(std::string version, bool create=true); + void reset(); +private: /// - /// \brief Expose the functionality this class is using - /// from the Mixin + /// \brief version_ /// + std::string version_; - using EnvMixin>::full_name; - using EnvMixin>::is_created; - using EnvMixin>::version; - using EnvMixin>::idx; - using EnvMixin>::py_env_name; - using EnvMixin>::py_reset_result_name; - using EnvMixin>::py_step_result_name; - using EnvMixin>::py_state_name; + /// + /// \brief init_mode_ + /// + GridworldInitType init_mode_; /// - /// \brief make. Builds the environment. Optionally we can choose if the - /// environment will be slippery + /// \brief is_created_ /// - void make(); + bool is_created_; /// - /// \brief n_states. Returns the number of states + /// \brief init_board_ /// - uint_t n_states()const noexcept{ return side_size == 4 ? 16 : 64; } + void init_board_(); /// - /// \brief n_actions. Returns the number of actions + /// \brief build_static_mode_ /// - uint_t n_actions()const noexcept{return action_space_type::size;} + void build_static_mode_(); /// - /// \brief step - /// \param action - /// \return + /// \brief build_random_mode_ /// - time_step_type step(action_type action, bool query_extra=false); + void build_random_mode_(); /// - /// \brief render + /// \brief build_player_mode_ /// - void render(); + void build_player_mode_(); - void close(); - void reset() + typedef std::pair Position; + + /// + /// \brief The BoardPiece struct + /// + struct BoardPiece + { + /// + /// + /// + std::string name; + /// + /// n ASCII character to display on the board + /// + std::string code; + + /// + /// \brief pos 2-tuple e.g. (1,4) + /// + Position pos; + + /// + /// \brief BoardPiece + /// \param name_ + /// \param code_ + /// \param pos_ + /// + BoardPiece(std::string name_, std::string code_, Position pos_) + : + name(name_), + code(code_), + pos(pos_) + {} + + /// + /// + /// + BoardPiece()=default; + }; + + /// + /// \brief The BoardPiece struct + /// + struct BoardMask + { + /// + /// + /// + std::string name; + /// + /// n ASCII character to display on the board + /// + std::string code; + + /// + /// \brief pos 2-tuple e.g. (1,4) + /// + Position pos; + + /// + /// \brief BoardPiece + /// \param name_ + /// \param code_ + /// \param pos_ + /// + BoardMask(std::string name_, std::string code_, Position pos_) + : + name(name_), + code(code_), + pos(pos_) + {} + }; + + /// + /// + /// + struct Board + { + uint_t size; + std::map components; + std::map masks; + }; + /// + /// \brief board_ + /// + Board board_; }; +template +const std::string Gridworld::name = "Gridworld"; + +template +Gridworld::Gridworld(std::string version, GridworldInitType init_mode, bool create) + : + version_(version), + init_mode_(init_mode), + is_created_(false) +{ + if(create){ + make(); + } +} + +template +void +Gridworld::make(){ + + if(is_created()){ + return; + } + + // initialize the board + init_board_(); + switch (init_mode_) { + + case GridworldInitType::STATIC: + { + build_static_mode_(); + break; + } + case GridworldInitType::RANDOM: + { + build_random_mode_(); + break; + } + case GridworldInitType::PLAYER: + { + build_player_mode_(); + break; + } +#ifdef GYMFCPP_DEBUG + default: + { + assert(false && "Invalid initialization mode"); + } +#endif + + } + + is_created_ = true; +} + +template +typename Gridworld::time_step_type +Gridworld::step(action_type action){ + +} + +template +void +Gridworld::render(){ + +} + +template +void +Gridworld::close(){ + +} + +template +void +Gridworld::reset(){ + +} + + +template +void +Gridworld::init_board_(){ + + board_.size = side_size; + board_.components["Player"] = BoardPiece("Player", "P", std::make_pair(0, 0)); + board_.components["Goal"] = BoardPiece("Goal", "G", std::make_pair(1, 0)); + board_.components["Pit"] = BoardPiece("Pit", "-", std::make_pair(2, 0)); + board_.components["Wall"] = BoardPiece("Wall", "W", std::make_pair(3, 0)); +} + +template +void +Gridworld::build_static_mode_(){ + + // Row, Column + board_.components["Player"].pos = std::make_pair(0,3); + board_.components["Goal"].pos = std::make_pair(0,0); + board_.components["Pit"].pos = std::make_pair(0,1); + board_.components["Wall"].pos = std::make_pair(1,1); + +} + +template +void +Gridworld::build_random_mode_(){ + +} + +template +void +Gridworld:: build_player_mode_(){ + +} + + } #endif // GRID_WORLD_ENV_H diff --git a/tests/test_grid_world/test_grid_world.cpp b/tests/test_grid_world/test_grid_world.cpp index 3df507de..d182f325 100644 --- a/tests/test_grid_world/test_grid_world.cpp +++ b/tests/test_grid_world/test_grid_world.cpp @@ -1,10 +1,10 @@ -#include "gymfcpp/frozen_lake_env.h" +#include "gymfcpp/grid_world_env.h" + #include "gymfcpp/time_step.h" #include "gymfcpp/time_step_type.h" #include "gymfcpp/gymfcpp_types.h" #include -#include namespace{ @@ -14,49 +14,48 @@ using gymfcpp::real_t; } -TEST(TestFrozenLake, TestConstructor4x4) { +TEST(TestGridworld, TestConstructor4x4) { try{ - Py_Initialize(); - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - gymfcpp::FrozenLake<4> env("v0", main_namespace); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); ASSERT_EQ(env.n_states(), static_cast(16)); ASSERT_EQ(env.n_actions(), static_cast(4)); - ASSERT_EQ(env.map_type(), "4x4"); + ASSERT_EQ(env.version(), "v0"); + ASSERT_FALSE(env.is_created()); + + // TODO: Think how to test this + ASSERT_TRUE(rlenvs::to_string(env.init_type()) == rlenvs::to_string(rlenvs::GridworldInitType::STATIC)); + ASSERT_EQ(env.name, "Gridworld"); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL(); + FAIL()<<"Unknown exception thrown"; } } -TEST(TestFrozenLake, TestConstructor8x8) { + +TEST(TestGridworld, TestMake) { try{ - Py_Initialize(); - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - gymfcpp::FrozenLake<8> env("v0", main_namespace); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); + env.make(); - ASSERT_EQ(env.n_states(), static_cast(64)); + ASSERT_TRUE(env.is_created()); + ASSERT_EQ(env.n_states(), static_cast(16)); ASSERT_EQ(env.n_actions(), static_cast(4)); - ASSERT_EQ(env.map_type(), "8x8"); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL(); + FAIL()<<"Unknown exception thrown"; } } -TEST(TestFrozenLake, Test_Make) +/*TEST(TestFrozenLake, Test_Make) { try{ @@ -202,4 +201,4 @@ TEST(TestFrozenLake, TestRender) FAIL()<<"Error could not step in the environment"; } } - +*/ From 703f84118e5660be25313457dd0364ed49128a1e Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 23 Feb 2022 22:15:08 +0000 Subject: [PATCH 4/5] #50 Update implementation --- src/gymfcpp/grid_world_env.h | 79 ++++++++++++++++++++++- tests/test_grid_world/test_grid_world.cpp | 44 ++++++------- 2 files changed, 98 insertions(+), 25 deletions(-) diff --git a/src/gymfcpp/grid_world_env.h b/src/gymfcpp/grid_world_env.h index e93fe068..81db2dfc 100644 --- a/src/gymfcpp/grid_world_env.h +++ b/src/gymfcpp/grid_world_env.h @@ -194,6 +194,21 @@ class Gridworld final /// void build_player_mode_(); + /// + /// \brief check_move_ + /// \param row + /// \param col + /// + void check_move_(uint_t row, uint_t col)const; + + /// + /// \brief validate_move_ + /// \param piece + /// \param row + /// \param col + /// + void validate_move_(const std::string piece, uint_t row, uint_t col); + typedef std::pair Position; @@ -230,7 +245,7 @@ class Gridworld final {} /// - /// + /// \brief Default constructor /// BoardPiece()=default; }; @@ -342,6 +357,62 @@ template typename Gridworld::time_step_type Gridworld::step(action_type action){ + // need to determine what object (if any) + // is in the new grid spot the player is moving to + // actions in {u,d,l,r} + //def checkMove(addpos): + // if self.validateMove('Player', addpos) in [0, 2]: + // new_pos = addTuple(self.board.components['Player'].pos, addpos) + // self.board.movePiece('Player', new_pos) + + switch( action ){ + case 0: + { + // move up + check_move_(-1, 0); + break; + } + case 1: + { + //down + check_move_(1, 0); + break; + } + case 2: + { + // left + check_move_(0, -1); + break; + } + case 3: + { + // right + check_move_(0, 1); + break; + } +#ifdef GYMFCPP_DEBUG + default: + { + assert(false && "Invalid move"); + } +#endif + + } + + + + /*if self.add_noise_on_state: + obs = self.board.render_np().reshape(1, 64) + np.random.rand(1, 64) / self.noise_factor + else: + obs = self.board.render_np().reshape(1, 64) + + reward = self.get_reward() + step_type = StepType.LAST if reward != -1 else StepType.MID + time_step = TimeStep(step_type=step_type, reward=reward, + info={}, observation=obs, discount=0.0) + return time_step*/ + + return time_step_type(); } template @@ -374,6 +445,12 @@ Gridworld::init_board_(){ board_.components["Wall"] = BoardPiece("Wall", "W", std::make_pair(3, 0)); } +template +void +Gridworld::check_move_(uint_t row, uint_t col)const{ + +} + template void Gridworld::build_static_mode_(){ diff --git a/tests/test_grid_world/test_grid_world.cpp b/tests/test_grid_world/test_grid_world.cpp index d182f325..cd5ad39f 100644 --- a/tests/test_grid_world/test_grid_world.cpp +++ b/tests/test_grid_world/test_grid_world.cpp @@ -55,55 +55,51 @@ TEST(TestGridworld, TestMake) { } } -/*TEST(TestFrozenLake, Test_Make) +TEST(TestGridworld, TestStepInvalidMove) { try{ - Py_Initialize(); - - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - - gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); env.make(); - //ASSERT_EQ(env.n_states(), static_cast(16)); - //ASSERT_EQ(env.n_actions(), static_cast(4)); + // make sure we have the right world + ASSERT_TRUE(env.is_created()); + ASSERT_EQ(env.n_states(), static_cast(16)); + ASSERT_EQ(env.n_actions(), static_cast(4)); + EXPECT_DEATH(env.step(5), "Invalid move"); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL(); + FAIL()<<"Unknown exception thrown"; } } -TEST(TestFrozenLake, Test_Reset) +TEST(TestGridworld, TestStepValidMove) { try{ - Py_Initialize(); - - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - - gymfcpp::FrozenLake<4> env("v0", main_namespace, false); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); env.make(); - auto state = env.reset(); - ASSERT_TRUE(state.first()); + // make sure we have the right world + ASSERT_TRUE(env.is_created()); + ASSERT_EQ(env.n_states(), static_cast(16)); + ASSERT_EQ(env.n_actions(), static_cast(4)); + + env.step(1); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL()<<"Error could not reset the environment"; + FAIL()<<"Unknown exception thrown"; } } +/* TEST(TestFrozenLake, Test_Step) { From 62041c2a9ff6fd8ee21d6381cf6171959331a742 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 26 Feb 2022 15:08:53 +0000 Subject: [PATCH 5/5] #50 Update API for Gridworld --- src/gymfcpp/grid_world_env.h | 374 +++++++++++++++++++--- src/gymfcpp/numpy_cpp_utils.h | 83 ++++- tests/test_grid_world/test_grid_world.cpp | 81 +++-- 3 files changed, 452 insertions(+), 86 deletions(-) diff --git a/src/gymfcpp/grid_world_env.h b/src/gymfcpp/grid_world_env.h index 81db2dfc..6fde065f 100644 --- a/src/gymfcpp/grid_world_env.h +++ b/src/gymfcpp/grid_world_env.h @@ -14,11 +14,14 @@ #include "gymfcpp/env_mixin.h" #include "gymfcpp/extern/enum.h" #include "gymfcpp/gymfcpp_config.h" +#include "gymfcpp/time_step_type.h" +#include "gymfcpp/numpy_cpp_utils.h" #ifdef GYMFCPP_DEBUG #include #endif +#include #include #include #include @@ -33,6 +36,36 @@ namespace rlenvs // still we may want to use some utilities using namespace gymfcpp; +namespace{ + +uint_t +max(const std::pair& position){ + return position.first > position.second ? position.first : position.second; +} + +uint_t +min(const std::pair& position){ + return position.first < position.second ? position.first : position.second; +} + +uint_t +get_non_zero_idx(std::pair& p){ + +} + +inline +bool operator == (const std::pair& p1, const std::pair& p2){ + return p1.first == p2.first && p1.second == p2.second; +} + +inline +std::pair +operator + (const std::pair& p1, const std::pair& p2){ + return std::make_pair(p1.first + p2.first, p1.second + p2.second); +} + +} + /// /// \brief The RenderModeType enum @@ -59,12 +92,12 @@ std::string to_string(GridworldInitType type){return type._to_string();} /// Player initialization means that the player is initialized at a random position on the board. /// Random initialization means that all the objects are placed randomly /// -template +template class Gridworld final { public: - static_assert (side_size >= 4, "The side size should be greater than 4"); + static_assert (side_size_ >= 4, "The side size should be greater than 4"); /// /// \brief name @@ -84,23 +117,43 @@ class Gridworld final /// /// \brief action_space_t. The type of the action space /// - typedef DiscreteSpace state_space_type; + typedef DiscreteSpace state_space_type; /// /// \brief state_type /// - typedef typename state_space_type::item_t state_type; + //typedef typename state_space_type::item_t state_type; + + typedef std::vector>> raw_state_type; + typedef std::vector state_type; /// /// \brief time_step_t. The type of the time step /// typedef TimeStep time_step_type; + /// + /// \brief n_components + /// + static const uint_t n_components = 4; + + /// + /// \brief side_size + /// + static const uint_t side_size = side_size_; + /// /// \brief Constructor /// Gridworld(std::string version, GridworldInitType init_mode, bool create=true); + /// + /// \brief Constructor. Initializes the world by allowing noise to be + /// added in the observation vector + /// + Gridworld(std::string version, GridworldInitType init_mode, uint_t seed, + real_t noise_factor, bool create=true); + /// /// \brief version /// \return @@ -128,7 +181,7 @@ class Gridworld final /// /// \brief n_states. Returns the number of states /// - uint_t n_states()const noexcept{ return side_size * side_size; } + uint_t n_states()const noexcept{ return side_size_ * side_size_; } /// /// \brief n_actions. Returns the number of actions @@ -153,9 +206,39 @@ class Gridworld final void close(); /// + /// \brief reset the environment + /// + time_step_type reset(); + /// + /// \brief get_reward + /// \return + /// + real_t get_reward()const; + + /// + /// \brief get_observation + /// \return + /// + raw_state_type get_raw_observation()const; + + /// + /// \brief has_random_state + /// \return /// - void reset(); + bool has_random_state()const noexcept{return randomize_state_;} + + /// + /// \brief seed + /// \return + /// + uint_t seed()const noexcept{return seed_;} + + /// + /// \brief noise_factor + /// \return + /// + real_t noise_factor()const noexcept{return noise_factor_;} private: @@ -169,11 +252,38 @@ class Gridworld final /// GridworldInitType init_mode_; + /// + /// \brief randomize_state_ + /// + bool randomize_state_; + + /// + /// \brief seed_ + /// + uint_t seed_; + + /// + /// \brief noise_factor_ + /// + real_t noise_factor_; + /// /// \brief is_created_ /// bool is_created_; + /// + /// \brief The MoveType enum + /// + enum MoveType{VALID=0, INVALID=1, LOST_GAME=2}; + + /// + /// \brief The BoardComponentType enum + /// + enum BoardComponentType{PLAYER=0, GOAL=1, PIT=2, WALL=3}; + + typedef std::pair Position; + /// /// \brief init_board_ /// @@ -199,7 +309,7 @@ class Gridworld final /// \param row /// \param col /// - void check_move_(uint_t row, uint_t col)const; + void check_move_(uint_t row, uint_t col); /// /// \brief validate_move_ @@ -207,10 +317,7 @@ class Gridworld final /// \param row /// \param col /// - void validate_move_(const std::string piece, uint_t row, uint_t col); - - - typedef std::pair Position; + MoveType validate_move_(BoardComponentType piece, Position pos); /// /// \brief The BoardPiece struct @@ -284,13 +391,26 @@ class Gridworld final }; /// - /// + /// Represents the board /// struct Board { uint_t size; - std::map components; + std::map components; std::map masks; + + /// + /// \brief move_piece Move the pice to the given position + /// \param piece + /// \param pos + /// + void move_piece(BoardComponentType piece, Position pos); + + /// + /// \brief get_observation + /// \return + /// + raw_state_type get_observation()const; }; /// @@ -302,11 +422,15 @@ class Gridworld final template const std::string Gridworld::name = "Gridworld"; + template Gridworld::Gridworld(std::string version, GridworldInitType init_mode, bool create) : version_(version), init_mode_(init_mode), + randomize_state_(false), + seed_(0), + noise_factor_(0.0), is_created_(false) { if(create){ @@ -314,6 +438,18 @@ Gridworld::Gridworld(std::string version, GridworldInitType init_mode } } +template +Gridworld::Gridworld(std::string version, GridworldInitType init_mode, uint_t seed, + real_t noise_factor, bool create) + : + version_(version), + init_mode_(init_mode), + randomize_state_(true), + seed_(seed), + noise_factor_(noise_factor), + is_created_(false) +{} + template void Gridworld::make(){ @@ -357,14 +493,6 @@ template typename Gridworld::time_step_type Gridworld::step(action_type action){ - // need to determine what object (if any) - // is in the new grid spot the player is moving to - // actions in {u,d,l,r} - //def checkMove(addpos): - // if self.validateMove('Player', addpos) in [0, 2]: - // new_pos = addTuple(self.board.components['Player'].pos, addpos) - // self.board.movePiece('Player', new_pos) - switch( action ){ case 0: { @@ -399,20 +527,68 @@ Gridworld::step(action_type action){ } + // get the observation + auto obs = to_1d_from_3d(n_components, side_size, side_size, get_raw_observation()); + + if(has_random_state()){ + + // add uniform noise and scale + add_uniform_noise(obs, this->seed(), noise_factor()); + } + + auto reward = get_reward(); + auto step_type = reward == -1.0 ? TimeStepTp::LAST : TimeStepTp::MID; + auto time_step = time_step_type(step_type, reward, obs); + + return time_step; +} + +template +typename Gridworld::time_step_type +Gridworld::reset(){ + + // reinitialize the board + init_board_(); + auto obs = to_1d_from_3d(n_components, side_size, side_size, get_raw_observation()); + + if(has_random_state()){ + // add uniform noise and scale + add_uniform_noise(obs, this->seed(), noise_factor()); + } - /*if self.add_noise_on_state: - obs = self.board.render_np().reshape(1, 64) + np.random.rand(1, 64) / self.noise_factor - else: - obs = self.board.render_np().reshape(1, 64) + auto reward = get_reward(); + auto time_step = time_step_type(TimeStepTp::FIRST, reward, obs); + return time_step; +} - reward = self.get_reward() - step_type = StepType.LAST if reward != -1 else StepType.MID - time_step = TimeStep(step_type=step_type, reward=reward, - info={}, observation=obs, discount=0.0) - return time_step*/ +template +real_t +Gridworld::get_reward()const{ + + auto player_pos = board_.components.find(BoardComponentType::PLAYER)->second.pos; + auto pit_pos = board_.components.find(BoardComponentType::PIT)->second.pos; + auto goal_pos = board_.components.find(BoardComponentType::GOAL)->second.pos; + if (player_pos == pit_pos){ + return -10.; + } + else if (player_pos == goal_pos){ + return 10.0; + } - return time_step_type(); + return -1.0; +} + +template +typename Gridworld::raw_state_type +Gridworld::get_raw_observation()const{ + +#ifdef GYMFCPP_DEBUG + if(!is_created_){ + assert(false && "Environment has not been created. Have you called make?"); + } +#endif + return board_.get_observation(); } template @@ -429,25 +605,61 @@ Gridworld::close(){ template void -Gridworld::reset(){ +Gridworld::init_board_(){ + board_.size = side_size; + board_.components[BoardComponentType::PLAYER] = BoardPiece("Player", "P", std::make_pair(0, 0)); + board_.components[BoardComponentType::GOAL] = BoardPiece("Goal", "G", std::make_pair(1, 0)); + board_.components[BoardComponentType::PIT] = BoardPiece("Pit", "-", std::make_pair(2, 0)); + board_.components[BoardComponentType::WALL] = BoardPiece("Wall", "W", std::make_pair(3, 0)); } template -void -Gridworld::init_board_(){ +typename Gridworld::MoveType +Gridworld::validate_move_(BoardComponentType piece, Position pos){ + + // 0 is valid + //auto outcome = 0 #0 is valid, 1 invalid, 2 lost game + auto outcome = Gridworld::MoveType::VALID; + + // get position of pit + auto pit_pos = board_.components[BoardComponentType::PIT].pos; + auto wall_pos = board_.components[BoardComponentType::WALL].pos; + + auto new_pos = board_.components[piece].pos + pos; + + if (new_pos == wall_pos){ + //1 //block move, player can't move to wall + outcome = Gridworld::MoveType::INVALID; + } + else if( max(new_pos) > (board_.size - 1 )){ + // #if outside bounds of board + outcome = Gridworld::MoveType::INVALID; + } + else if( min(new_pos) < 0){ + // #if outside bounds + outcome = Gridworld::MoveType::INVALID; + } + else if( new_pos == pit_pos){ + outcome = Gridworld::MoveType::LOST_GAME; + } - board_.size = side_size; - board_.components["Player"] = BoardPiece("Player", "P", std::make_pair(0, 0)); - board_.components["Goal"] = BoardPiece("Goal", "G", std::make_pair(1, 0)); - board_.components["Pit"] = BoardPiece("Pit", "-", std::make_pair(2, 0)); - board_.components["Wall"] = BoardPiece("Wall", "W", std::make_pair(3, 0)); + return outcome; } template void -Gridworld::check_move_(uint_t row, uint_t col)const{ +Gridworld::check_move_(uint_t row, uint_t col){ + + auto position = std::make_pair(row, col); + auto move_type = validate_move_(BoardComponentType::PLAYER, position); + + if( move_type == MoveType::VALID || move_type == MoveType::LOST_GAME){ + + auto new_pos = board_.components[BoardComponentType::PLAYER].pos + position; + board_.move_piece(BoardComponentType::PLAYER, new_pos); + } } @@ -456,10 +668,10 @@ void Gridworld::build_static_mode_(){ // Row, Column - board_.components["Player"].pos = std::make_pair(0,3); - board_.components["Goal"].pos = std::make_pair(0,0); - board_.components["Pit"].pos = std::make_pair(0,1); - board_.components["Wall"].pos = std::make_pair(1,1); + board_.components[BoardComponentType::PLAYER].pos = std::make_pair(0,3); + board_.components[BoardComponentType::GOAL].pos = std::make_pair(0,0); + board_.components[BoardComponentType::PIT].pos = std::make_pair(0,1); + board_.components[BoardComponentType::WALL].pos = std::make_pair(1,1); } @@ -475,6 +687,80 @@ Gridworld:: build_player_mode_(){ } +// Board inner struct + +template +typename Gridworld::raw_state_type +Gridworld::Board::get_observation()const{ + + // initialize the data struct the + // represents the state + auto num_pieces = components.size() + masks.size(); + raw_state_type board(num_pieces); + + for(uint_t i=0; isecond.pos; + board[layer][position.first][position.second] = 1; + layer +=1; + } + + return board; + + // initialize the data struct the + // represents the state + /* displ_board = np.zeros((num_pieces, self.size, self.size), dtype=np.uint8) + layer = 0 + for name, piece in self.components.items(): + pos = (layer,) + piece.pos + displ_board[pos] = 1 + layer += 1*/ + + +} + +template +void +Gridworld::Board::move_piece(BoardComponentType piece, Position pos){ + + auto move = true; + + // check if we can move the piece + auto mask_begin = masks.begin(); + auto mask_end = masks.end(); + + /*for(; mask_begin != mask_end; ++mask_begin){ + position = + }*/ + + + // for _, mask in self.masks.items(): + // if pos in zip_positions2d(mask.get_positions()): + // move = False + if( move){ + components[piece].pos = pos; + } + +} + } diff --git a/src/gymfcpp/numpy_cpp_utils.h b/src/gymfcpp/numpy_cpp_utils.h index a06c19d5..91983894 100644 --- a/src/gymfcpp/numpy_cpp_utils.h +++ b/src/gymfcpp/numpy_cpp_utils.h @@ -8,6 +8,7 @@ #include #endif +#include #include namespace gymfcpp{ @@ -59,7 +60,8 @@ std::vector linspace(T start_in, T end_in, uint_t num_in) /// The SerialContainer is expected to be sorted in ascending order. /// template -uint_t digitize(const T x, const SerialContainer& container){ +uint_t +digitize(const T x, const SerialContainer& container){ #ifdef GYMFCPP_DEBUG assert(container.empty() == false && "The given container is empty"); @@ -88,6 +90,85 @@ uint_t digitize(const T x, const SerialContainer& container){ return bin; } +/// +/// +/// +template +std::vector +to_1d_from_2d(uint_t dim1, uint_t dim2, const SerialVector& v){ + +#ifdef GYMFCPP_DEBUG + assert(dim1 == v.size() && "Invalid vector first dimension != dim1"); + assert(dim2 == v[0].size() && "Invalid vector second dimension != dim2"); +#endif + + std::vector result(dim1 * dim2); + auto counter = 0; + for(uint_t i=0; i +std::vector +to_1d_from_3d(uint_t dim1, uint_t dim2, uint_t dim3, const SerialVector& v){ + +#ifdef GYMFCPP_DEBUG + assert(dim1 == v.size() && "Invalid vector first dimension != dim1"); + assert(dim2 == v[0].size() && "Invalid vector second dimension != dim2"); + assert(dim3 == v[0][0].size() && "Invalid vector third dimension != dim3"); +#endif + + std::vector result(dim1 * dim2 * dim2); + auto counter = 0; + for(uint_t i=0; i +void +add_uniform_noise(SerialVector& vector, uint_t seed){ + + std::mt19937 gen(seed); + std::uniform_real_distribution<> real_dist_(0.0, 1.0); + + for(uint_t i=0; i +void +add_uniform_noise(SerialVector& vector, uint_t seed, real_t scale){ + + + add_uniform_noise(vector, seed); + + for(uint_t i=0; i(16)); ASSERT_EQ(env.n_actions(), static_cast(4)); + // this should always succeed env.step(1); } @@ -99,81 +100,79 @@ TEST(TestGridworld, TestStepValidMove) } } -/* -TEST(TestFrozenLake, Test_Step) -{ +TEST(TestGridworld, TestGetObservationFail) +{ try{ - Py_Initialize(); - - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - - gymfcpp::FrozenLake<4> env("v0", main_namespace, false); - env.make(); - - auto step_result = env.step(0); - ASSERT_TRUE(step_result.mid()); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); + EXPECT_DEATH(env.get_raw_observation(), "Environment has not been created. Have you called make?"); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL()<<"Error could not step in the environment"; + + FAIL()<<"Unknown exception thrown"; } } -TEST(TestFrozenLake, Test_Step_With_Query) + +TEST(TestGridworld, TestGetObservation) { try{ - Py_Initialize(); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, false); + env.make(); - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); + // make sure we have the right world + ASSERT_TRUE(env.is_created()); + ASSERT_EQ(env.n_states(), static_cast(16)); + ASSERT_EQ(env.n_actions(), static_cast(4)); - gymfcpp::FrozenLake<4> env("v0", main_namespace, false); - env.make(); + auto state = env.get_raw_observation(); + + ASSERT_EQ(state.size(), static_cast(env.n_components)); - auto step_result = env.step(0, true); - ASSERT_TRUE(step_result.mid()); - ASSERT_DOUBLE_EQ(step_result.get_extra("prob"), 0.3333333333333333); + for(uint_t c=0; c< env.n_components; ++c){ + ASSERT_EQ(state[c].size(), static_cast(env.side_size)); + } } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL()<<"Error could not step in the environment"; + FAIL()<<"Unknown exception thrown"; } } -TEST(TestFrozenLake, Test_Get_Dynamics) +TEST(TestGridworld, TestConstructor4x4Random) { try{ - Py_Initialize(); - - auto main_module = boost::python::import("__main__"); - auto main_namespace = main_module.attr("__dict__"); - - gymfcpp::FrozenLake<4> env("v0", main_namespace, false); - env.make(); + rlenvs::Gridworld<4> env("v0", rlenvs::GridworldInitType::STATIC, 42, 10.0, false); - auto dynamics = env.p(1, 3); + ASSERT_EQ(env.n_states(), static_cast(16)); + ASSERT_EQ(env.n_actions(), static_cast(4)); + ASSERT_EQ(env.version(), "v0"); + ASSERT_FALSE(env.is_created()); + ASSERT_TRUE(env.has_random_state()); + ASSERT_EQ(env.seed(), 42); + ASSERT_EQ(env.noise_factor(), 10.0); - ASSERT_EQ(dynamics.size(), static_cast(3) ); + // TODO: Think how to test this + ASSERT_TRUE(rlenvs::to_string(env.init_type()) == rlenvs::to_string(rlenvs::GridworldInitType::STATIC)); + ASSERT_EQ(env.name, "Gridworld"); } - catch(const boost::python::error_already_set&) + catch(...) { - PyErr_Print(); - FAIL()<<"Error could not step in the environment"; + + FAIL()<<"Unknown exception thrown"; } } +/* TEST(TestFrozenLake, TestRender) {