From fca3dcc900451e3e8f6acd2112fb2c791435fbfc Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 15 Jul 2019 02:15:36 +0430 Subject: [PATCH 1/6] Added files and changed CMakeLists.txt --- CMakeLists.txt | 17 +++++++++-------- torchvision/csrc/datasets/caltech.cpp | 5 +++++ torchvision/csrc/datasets/caltech.h | 11 +++++++++++ torchvision/csrc/datasets/datasets.h | 6 ++++++ torchvision/csrc/datasets/datasetsimpl.h | 4 ++++ 5 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 torchvision/csrc/datasets/caltech.cpp create mode 100644 torchvision/csrc/datasets/caltech.h create mode 100644 torchvision/csrc/datasets/datasets.h create mode 100644 torchvision/csrc/datasets/datasetsimpl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d1609f8e820..b4030b60666 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,21 +4,22 @@ set(CMAKE_CXX_STANDARD 11) find_package(Torch REQUIRED) -file(GLOB_RECURSE HEADERS torchvision/csrc/vision.h) -file(GLOB_RECURSE MODELS_HEADERS torchvision/csrc/models/*.h) -file(GLOB_RECURSE MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp) +file(GLOB HEADERS torchvision/csrc/vision.h) -add_library (${PROJECT_NAME} SHARED ${MODELS_SOURCES}) +file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h) +file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp) + +file(GLOB DATASETS_HEADERS torchvision/csrc/datasets/*.h) +file(GLOB DATASETS_SOURCES torchvision/csrc/datasets/*.h torchvision/csrc/datasets/*.cpp) + +add_library (${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${DATASETS_SOURCES}) target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") add_executable(convertmodels torchvision/csrc/convert_models/convert_models.cpp) target_link_libraries(convertmodels "${PROJECT_NAME}") target_link_libraries(convertmodels "${TORCH_LIBRARIES}") -#add_executable(testmodels test/test_models.cpp) -#target_link_libraries(testmodels "${PROJECT_NAME}") -#target_link_libraries(testmodels "${TORCH_LIBRARIES}") - install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}) install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/models) +install(FILES ${DATASETS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/datasets) diff --git a/torchvision/csrc/datasets/caltech.cpp b/torchvision/csrc/datasets/caltech.cpp new file mode 100644 index 00000000000..6c9362507a1 --- /dev/null +++ b/torchvision/csrc/datasets/caltech.cpp @@ -0,0 +1,5 @@ +#include "caltech.h" + +#include "datasetsimpl.h" + +Caltech101::Caltech101() {} diff --git a/torchvision/csrc/datasets/caltech.h b/torchvision/csrc/datasets/caltech.h new file mode 100644 index 00000000000..465107bcd8f --- /dev/null +++ b/torchvision/csrc/datasets/caltech.h @@ -0,0 +1,11 @@ +#ifndef CALTECH_H +#define CALTECH_H + +#include + +struct Caltech101 : torch::data::Dataset +{ + Caltech101(); +}; + +#endif // CALTECH_H diff --git a/torchvision/csrc/datasets/datasets.h b/torchvision/csrc/datasets/datasets.h new file mode 100644 index 00000000000..3a894fbaa7d --- /dev/null +++ b/torchvision/csrc/datasets/datasets.h @@ -0,0 +1,6 @@ +#ifndef DATASETS_H +#define DATASETS_H + +#include "caltech.h" + +#endif // DATASETS_H diff --git a/torchvision/csrc/datasets/datasetsimpl.h b/torchvision/csrc/datasets/datasetsimpl.h new file mode 100644 index 00000000000..615d6796fa5 --- /dev/null +++ b/torchvision/csrc/datasets/datasetsimpl.h @@ -0,0 +1,4 @@ +#ifndef DATASETSIMPL_H +#define DATASETSIMPL_H + +#endif // DATASETSIMPL_H From 225edb8a8c51b256ed042a121d68a0399cbf07f5 Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 15 Jul 2019 17:38:17 +0430 Subject: [PATCH 2/6] Wrote some stuff --- CMakeLists.txt | 9 +++- test.cpp | 15 ++++++ torchvision/csrc/datasets/caltech.cpp | 22 +++++++- torchvision/csrc/datasets/caltech.h | 17 +++++-- torchvision/csrc/datasets/datasets.h | 2 +- torchvision/csrc/datasets/datasetsimpl.cpp | 1 + torchvision/csrc/datasets/datasetsimpl.h | 59 ++++++++++++++++++++++ 7 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 test.cpp create mode 100644 torchvision/csrc/datasets/datasetsimpl.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b4030b60666..ac009f8c76a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 2.8) project(torchvision) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) @@ -13,8 +13,14 @@ file(GLOB DATASETS_HEADERS torchvision/csrc/datasets/*.h) file(GLOB DATASETS_SOURCES torchvision/csrc/datasets/*.h torchvision/csrc/datasets/*.cpp) add_library (${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${DATASETS_SOURCES}) +target_link_libraries(${PROJECT_NAME} stdc++fs) target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") +add_executable(test test.cpp) +target_link_libraries(test stdc++fs) +target_link_libraries(test "${PROJECT_NAME}") +target_link_libraries(test "${TORCH_LIBRARIES}") + add_executable(convertmodels torchvision/csrc/convert_models/convert_models.cpp) target_link_libraries(convertmodels "${PROJECT_NAME}") target_link_libraries(convertmodels "${TORCH_LIBRARIES}") @@ -23,3 +29,4 @@ install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}) install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/models) install(FILES ${DATASETS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/datasets) + diff --git a/test.cpp b/test.cpp new file mode 100644 index 00000000000..aba9c869ede --- /dev/null +++ b/test.cpp @@ -0,0 +1,15 @@ +#include "torchvision/csrc/datasets/datasets.h" +#include "torchvision/csrc/datasets/datasetsimpl.h" + +using namespace std; +using namespace vision::datasets; +using namespace vision::datasets::datasetsimpl; + +int main() { + torch::nn::Sequential( + torch::nn::Sequential(), + torch::nn::Sequential(), + torch::nn::Sequential()); + auto T = torch::tensor(10); + cout << T << endl; +} diff --git a/torchvision/csrc/datasets/caltech.cpp b/torchvision/csrc/datasets/caltech.cpp index 6c9362507a1..a0699bcad22 100644 --- a/torchvision/csrc/datasets/caltech.cpp +++ b/torchvision/csrc/datasets/caltech.cpp @@ -2,4 +2,24 @@ #include "datasetsimpl.h" -Caltech101::Caltech101() {} +namespace vision { +namespace datasets { + +Caltech101::Caltech101(const std::string& root) : root(root) { + TORCH_CHECK( + datasetsimpl::mkpath(root), + "Failed to create directory \"", + root, + "\" for Caltech101 dataset"); + + TORCH_CHECK(checkIntegrity(), "Caltech101 dataset not found or corrupted.") + + auto categories = datasetsimpl::lsdir(datasetsimpl::join(root, "caltech101")); +} + +bool Caltech101::checkIntegrity() { + return datasetsimpl::exists(datasetsimpl::join(root, "caltech101")); +} + +} // namespace datasets +} // namespace vision diff --git a/torchvision/csrc/datasets/caltech.h b/torchvision/csrc/datasets/caltech.h index 465107bcd8f..2df03facfee 100644 --- a/torchvision/csrc/datasets/caltech.h +++ b/torchvision/csrc/datasets/caltech.h @@ -3,9 +3,18 @@ #include -struct Caltech101 : torch::data::Dataset -{ - Caltech101(); +namespace vision { +namespace datasets { + +struct Caltech101 : torch::data::Dataset { + std::string root; + + Caltech101(const std::string& root); + + bool checkIntegrity(); }; -#endif // CALTECH_H +} // namespace datasets +} // namespace vision + +#endif // CALTECH_H diff --git a/torchvision/csrc/datasets/datasets.h b/torchvision/csrc/datasets/datasets.h index 3a894fbaa7d..eeebedd0b40 100644 --- a/torchvision/csrc/datasets/datasets.h +++ b/torchvision/csrc/datasets/datasets.h @@ -3,4 +3,4 @@ #include "caltech.h" -#endif // DATASETS_H +#endif // DATASETS_H diff --git a/torchvision/csrc/datasets/datasetsimpl.cpp b/torchvision/csrc/datasets/datasetsimpl.cpp new file mode 100644 index 00000000000..f400eff0f0d --- /dev/null +++ b/torchvision/csrc/datasets/datasetsimpl.cpp @@ -0,0 +1 @@ +#include "datasetsimpl.h" diff --git a/torchvision/csrc/datasets/datasetsimpl.h b/torchvision/csrc/datasets/datasetsimpl.h index 615d6796fa5..19f9680d973 100644 --- a/torchvision/csrc/datasets/datasetsimpl.h +++ b/torchvision/csrc/datasets/datasetsimpl.h @@ -1,4 +1,63 @@ #ifndef DATASETSIMPL_H #define DATASETSIMPL_H +#include +#include + +#ifndef TORCH_CHECK +#define TORCH_CHECK AT_CHECK +#endif + +namespace vision { +namespace datasets { +namespace datasetsimpl { + +inline std::vector lsdir(const std::string& path) { + std::vector list; + for (const auto& ent : std::filesystem::directory_iterator(path)) + list.push_back(ent.path().filename()); + return list; +} + +inline std::string tolower(std::string str) { + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + return str; +} + +inline void sort_names(std::vector& data) { + auto comp = [](const std::string& A, const std::string& B) { + return tolower(A) < tolower(B); + }; + std::sort(data.begin(), data.end(), comp); +} + +inline bool isdir(const std::string& path) { + return std::filesystem::is_directory(path); +} + +inline bool isfile(const std::string& path) { + return std::filesystem::is_regular_file(path); +} + +inline bool exists(const std::string& path) { + return std::filesystem::exists(path); +} + +inline bool mkpath(const std::string& path) { + return std::filesystem::create_directories(path); +} + +inline std::string join(const std::string& str) { + return str; +} + +template +inline std::string join(const std::string& head, Tail&&... tail) { + return std::filesystem::path(head).append(join(tail...)).string(); +} + +} // namespace datasetsimpl +} // namespace datasets +} // namespace vision + #endif // DATASETSIMPL_H From c53dd976adef096e70bfb3cede16f08da44df107 Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 15 Jul 2019 20:48:44 +0430 Subject: [PATCH 3/6] Completed caltech101 --- CMakeLists.txt | 9 +-- test.cpp | 15 ----- torchvision/csrc/datasets/caltech.cpp | 49 ++++++++++++-- torchvision/csrc/datasets/caltech.h | 21 +++++- torchvision/csrc/datasets/datasets.h | 10 +++ torchvision/csrc/datasets/datasetsimpl.cpp | 76 ++++++++++++++++++++++ torchvision/csrc/datasets/datasetsimpl.h | 51 +++++++-------- torchvision/csrc/{models => }/general.h | 0 torchvision/csrc/models/alexnet.h | 2 +- torchvision/csrc/models/densenet.h | 2 +- torchvision/csrc/models/googlenet.h | 2 +- torchvision/csrc/models/inception.h | 2 +- torchvision/csrc/models/mnasnet.h | 2 +- torchvision/csrc/models/mobilenet.h | 2 +- torchvision/csrc/models/resnet.h | 2 +- torchvision/csrc/models/shufflenetv2.h | 2 +- torchvision/csrc/models/squeezenet.h | 2 +- torchvision/csrc/models/vgg.h | 2 +- 18 files changed, 183 insertions(+), 68 deletions(-) delete mode 100644 test.cpp rename torchvision/csrc/{models => }/general.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac009f8c76a..f06fd494500 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,9 @@ project(torchvision) set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) +find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs) -file(GLOB HEADERS torchvision/csrc/vision.h) +file(GLOB HEADERS torchvision/csrc/vision.h torchvision/csrc/general.h) file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h) file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp) @@ -14,13 +15,9 @@ file(GLOB DATASETS_SOURCES torchvision/csrc/datasets/*.h torchvision/csrc/datase add_library (${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${DATASETS_SOURCES}) target_link_libraries(${PROJECT_NAME} stdc++fs) +target_link_libraries(${PROJECT_NAME} "${OpenCV_LIBS}") target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") -add_executable(test test.cpp) -target_link_libraries(test stdc++fs) -target_link_libraries(test "${PROJECT_NAME}") -target_link_libraries(test "${TORCH_LIBRARIES}") - add_executable(convertmodels torchvision/csrc/convert_models/convert_models.cpp) target_link_libraries(convertmodels "${PROJECT_NAME}") target_link_libraries(convertmodels "${TORCH_LIBRARIES}") diff --git a/test.cpp b/test.cpp deleted file mode 100644 index aba9c869ede..00000000000 --- a/test.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "torchvision/csrc/datasets/datasets.h" -#include "torchvision/csrc/datasets/datasetsimpl.h" - -using namespace std; -using namespace vision::datasets; -using namespace vision::datasets::datasetsimpl; - -int main() { - torch::nn::Sequential( - torch::nn::Sequential(), - torch::nn::Sequential(), - torch::nn::Sequential()); - auto T = torch::tensor(10); - cout << T << endl; -} diff --git a/torchvision/csrc/datasets/caltech.cpp b/torchvision/csrc/datasets/caltech.cpp index a0699bcad22..c1f3f0b642a 100644 --- a/torchvision/csrc/datasets/caltech.cpp +++ b/torchvision/csrc/datasets/caltech.cpp @@ -1,11 +1,15 @@ #include "caltech.h" -#include "datasetsimpl.h" - namespace vision { namespace datasets { -Caltech101::Caltech101(const std::string& root) : root(root) { +Caltech101::Caltech101( + std::string root, + std::function transform) + : cv_transform(transform) { + root = datasetsimpl::absolute_path(datasetsimpl::join(root, "caltech101")); + this->root = root; + TORCH_CHECK( datasetsimpl::mkpath(root), "Failed to create directory \"", @@ -14,11 +18,46 @@ Caltech101::Caltech101(const std::string& root) : root(root) { TORCH_CHECK(checkIntegrity(), "Caltech101 dataset not found or corrupted.") - auto categories = datasetsimpl::lsdir(datasetsimpl::join(root, "caltech101")); + categories = + datasetsimpl::lsdir(datasetsimpl::join(root, "101_ObjectCategories")); + datasetsimpl::sort_names(categories); + + auto it = + std::find(categories.begin(), categories.end(), "BACKGROUND_Google"); + if (it != categories.end()) + categories.erase(it); + + long count = 0; + for (const auto& category : categories) { + auto files = datasetsimpl::lsdir( + datasetsimpl::join(root, "101_ObjectCategories", category)); + + for (auto& file : files) + data.emplace_back(std::make_pair(std::move(file), count)); + + ++count; + } +} + +Caltech101::Example Caltech101::get(size_t index) { + auto& pair = data[index]; + auto path = datasetsimpl::join( + root, + "101_ObjectCategories", + categories[size_t(pair.second)], + pair.first); + + auto data = datasetsimpl::read_image(path, cv_transform); + auto target = torch::from_blob(&pair.second, {1}, torch::kLong).clone(); + return {data, target}; +} + +torch::optional Caltech101::size() const { + return data.size(); } bool Caltech101::checkIntegrity() { - return datasetsimpl::exists(datasetsimpl::join(root, "caltech101")); + return datasetsimpl::exists(datasetsimpl::join(root, "101_ObjectCategories")); } } // namespace datasets diff --git a/torchvision/csrc/datasets/caltech.h b/torchvision/csrc/datasets/caltech.h index 2df03facfee..b53d489bb36 100644 --- a/torchvision/csrc/datasets/caltech.h +++ b/torchvision/csrc/datasets/caltech.h @@ -2,16 +2,31 @@ #define CALTECH_H #include +#include "../general.h" +#include "datasetsimpl.h" namespace vision { namespace datasets { -struct Caltech101 : torch::data::Dataset { - std::string root; +class VISION_API Caltech101 : torch::data::Dataset { + using Example = torch::data::Example<>; - Caltech101(const std::string& root); + std::string root; + std::function cv_transform; + std::vector categories; + std::vector> data; bool checkIntegrity(); + + public: + Caltech101( + std::string root, + std::function cv_transform = + datasetsimpl::rgb_transform); + + Example get(size_t index) override; + + torch::optional size() const override; }; } // namespace datasets diff --git a/torchvision/csrc/datasets/datasets.h b/torchvision/csrc/datasets/datasets.h index eeebedd0b40..b2355b2664c 100644 --- a/torchvision/csrc/datasets/datasets.h +++ b/torchvision/csrc/datasets/datasets.h @@ -3,4 +3,14 @@ #include "caltech.h" +namespace vision { +namespace datasets { +namespace cv_transforms { +using datasetsimpl::gray_transform; +using datasetsimpl::make_transform; +using datasetsimpl::rgb_transform; +} // namespace cv_transforms +} // namespace datasets +} // namespace vision + #endif // DATASETS_H diff --git a/torchvision/csrc/datasets/datasetsimpl.cpp b/torchvision/csrc/datasets/datasetsimpl.cpp index f400eff0f0d..65bef70a017 100644 --- a/torchvision/csrc/datasets/datasetsimpl.cpp +++ b/torchvision/csrc/datasets/datasetsimpl.cpp @@ -1 +1,77 @@ #include "datasetsimpl.h" + +std::vector vision::datasets::datasetsimpl::lsdir( + const std::string& path) { + std::vector list; + for (const auto& ent : std::filesystem::directory_iterator(path)) + list.push_back(ent.path().filename()); + return list; +} + +std::string vision::datasets::datasetsimpl::tolower(std::string str) { + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + return str; +} + +void vision::datasets::datasetsimpl::sort_names( + std::vector& data) { + auto comp = [](const std::string& A, const std::string& B) { + return tolower(A) < tolower(B); + }; + std::sort(data.begin(), data.end(), comp); +} + +bool vision::datasets::datasetsimpl::isdir(const std::string& path) { + return std::filesystem::is_directory(path); +} + +bool vision::datasets::datasetsimpl::isfile(const std::string& path) { + return std::filesystem::is_regular_file(path); +} + +bool vision::datasets::datasetsimpl::exists(const std::string& path) { + return std::filesystem::exists(path); +} + +bool vision::datasets::datasetsimpl::mkpath(const std::string& path) { + if (exists(path)) + return true; + return std::filesystem::create_directories(path); +} + +torch::Tensor vision::datasets::datasetsimpl::read_image( + const std::string& path, + std::function transform) { + auto mat = cv::imread(path); + TORCH_CHECK(!mat.empty(), "Failed to read image \"", path, "\"."); + + // TODO make channels variable + + mat = transform(mat); + std::vector tensors; + std::vector channels(size_t(mat.channels())); + cv::split(mat, channels); + + for (auto& channel : channels) + tensors.push_back( + torch::from_blob(channel.ptr(), {mat.rows, mat.cols}, torch::kUInt8)); + + return torch::cat(tensors) + .view({mat.channels(), mat.rows, mat.cols}) + .to(torch::kFloat); +} + +std::string vision::datasets::datasetsimpl::absolute_path( + const std::string& path) { + return std::filesystem::absolute(path).string(); +} + +std::function vision::datasets::datasetsimpl:: + make_transform(int width, int height, cv::ColorConversionCodes code) { + return [width, height, code](const cv::Mat& mat) { + cv::Mat new_mat; + cv::resize(mat, new_mat, cv::Size(width, height)); + cv::cvtColor(new_mat, new_mat, code); + return new_mat; + }; +} diff --git a/torchvision/csrc/datasets/datasetsimpl.h b/torchvision/csrc/datasets/datasetsimpl.h index 19f9680d973..5f72b1c8e3f 100644 --- a/torchvision/csrc/datasets/datasetsimpl.h +++ b/torchvision/csrc/datasets/datasetsimpl.h @@ -1,6 +1,7 @@ #ifndef DATASETSIMPL_H #define DATASETSIMPL_H +#include #include #include @@ -12,50 +13,42 @@ namespace vision { namespace datasets { namespace datasetsimpl { -inline std::vector lsdir(const std::string& path) { - std::vector list; - for (const auto& ent : std::filesystem::directory_iterator(path)) - list.push_back(ent.path().filename()); - return list; -} +std::vector lsdir(const std::string& path); -inline std::string tolower(std::string str) { - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - return str; -} +std::string tolower(std::string str); -inline void sort_names(std::vector& data) { - auto comp = [](const std::string& A, const std::string& B) { - return tolower(A) < tolower(B); - }; - std::sort(data.begin(), data.end(), comp); -} +void sort_names(std::vector& data); -inline bool isdir(const std::string& path) { - return std::filesystem::is_directory(path); -} +bool isdir(const std::string& path); -inline bool isfile(const std::string& path) { - return std::filesystem::is_regular_file(path); -} +bool isfile(const std::string& path); -inline bool exists(const std::string& path) { - return std::filesystem::exists(path); -} +bool exists(const std::string& path); -inline bool mkpath(const std::string& path) { - return std::filesystem::create_directories(path); -} +bool mkpath(const std::string& path); + +std::string absolute_path(const std::string& path); inline std::string join(const std::string& str) { return str; } - template inline std::string join(const std::string& head, Tail&&... tail) { return std::filesystem::path(head).append(join(tail...)).string(); } +torch::Tensor read_image( + const std::string& path, + std::function transform); + +std::function make_transform( + int width, + int height, + cv::ColorConversionCodes code); + +inline auto rgb_transform = make_transform(224, 224, cv::COLOR_BGR2RGB); +inline auto gray_transform = make_transform(224, 224, cv::COLOR_BGR2GRAY); + } // namespace datasetsimpl } // namespace datasets } // namespace vision diff --git a/torchvision/csrc/models/general.h b/torchvision/csrc/general.h similarity index 100% rename from torchvision/csrc/models/general.h rename to torchvision/csrc/general.h diff --git a/torchvision/csrc/models/alexnet.h b/torchvision/csrc/models/alexnet.h index 813531fb383..4c164da64d0 100644 --- a/torchvision/csrc/models/alexnet.h +++ b/torchvision/csrc/models/alexnet.h @@ -2,7 +2,7 @@ #define ALEXNET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/densenet.h b/torchvision/csrc/models/densenet.h index 3ed6eba0837..f758b70ab1e 100644 --- a/torchvision/csrc/models/densenet.h +++ b/torchvision/csrc/models/densenet.h @@ -2,7 +2,7 @@ #define DENSENET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/googlenet.h b/torchvision/csrc/models/googlenet.h index 94390fd5070..4bf334939ac 100644 --- a/torchvision/csrc/models/googlenet.h +++ b/torchvision/csrc/models/googlenet.h @@ -2,7 +2,7 @@ #define GOOGLENET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/inception.h b/torchvision/csrc/models/inception.h index d4edcbadd47..c01bfb5c0b8 100644 --- a/torchvision/csrc/models/inception.h +++ b/torchvision/csrc/models/inception.h @@ -2,7 +2,7 @@ #define INCEPTION_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/mnasnet.h b/torchvision/csrc/models/mnasnet.h index e499a7df987..86ae177f4a7 100644 --- a/torchvision/csrc/models/mnasnet.h +++ b/torchvision/csrc/models/mnasnet.h @@ -2,7 +2,7 @@ #define MNASNET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/mobilenet.h b/torchvision/csrc/models/mobilenet.h index e3a498b08ea..3bb5ae9c76c 100644 --- a/torchvision/csrc/models/mobilenet.h +++ b/torchvision/csrc/models/mobilenet.h @@ -2,7 +2,7 @@ #define MOBILENET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/resnet.h b/torchvision/csrc/models/resnet.h index b9ef32f7956..4fafd1c0c7c 100644 --- a/torchvision/csrc/models/resnet.h +++ b/torchvision/csrc/models/resnet.h @@ -2,7 +2,7 @@ #define RESNET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/shufflenetv2.h b/torchvision/csrc/models/shufflenetv2.h index 0e357f2ce7d..000e4a2b37a 100644 --- a/torchvision/csrc/models/shufflenetv2.h +++ b/torchvision/csrc/models/shufflenetv2.h @@ -2,7 +2,7 @@ #define SHUFFLENETV2_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/squeezenet.h b/torchvision/csrc/models/squeezenet.h index 298f1f04095..8b74e056f43 100644 --- a/torchvision/csrc/models/squeezenet.h +++ b/torchvision/csrc/models/squeezenet.h @@ -2,7 +2,7 @@ #define SQUEEZENET_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { diff --git a/torchvision/csrc/models/vgg.h b/torchvision/csrc/models/vgg.h index cc9f98aea77..e48c1d078b3 100644 --- a/torchvision/csrc/models/vgg.h +++ b/torchvision/csrc/models/vgg.h @@ -2,7 +2,7 @@ #define VGG_H #include -#include "general.h" +#include "../general.h" namespace vision { namespace models { From b47430f523c8c610c06aaeae2ff799f8d369afc3 Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 15 Jul 2019 23:12:51 +0430 Subject: [PATCH 4/6] Changed filesystem to posix --- CMakeLists.txt | 3 +- torchvision/csrc/datasets/caltech.cpp | 6 --- torchvision/csrc/datasets/datasetsimpl.cpp | 46 +++++++++++++++------- torchvision/csrc/datasets/datasetsimpl.h | 9 ++--- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f06fd494500..f0f116eeb16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 2.8) project(torchvision) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 11) find_package(Torch REQUIRED) find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs) @@ -14,7 +14,6 @@ file(GLOB DATASETS_HEADERS torchvision/csrc/datasets/*.h) file(GLOB DATASETS_SOURCES torchvision/csrc/datasets/*.h torchvision/csrc/datasets/*.cpp) add_library (${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${DATASETS_SOURCES}) -target_link_libraries(${PROJECT_NAME} stdc++fs) target_link_libraries(${PROJECT_NAME} "${OpenCV_LIBS}") target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") diff --git a/torchvision/csrc/datasets/caltech.cpp b/torchvision/csrc/datasets/caltech.cpp index c1f3f0b642a..b9fbb428449 100644 --- a/torchvision/csrc/datasets/caltech.cpp +++ b/torchvision/csrc/datasets/caltech.cpp @@ -10,12 +10,6 @@ Caltech101::Caltech101( root = datasetsimpl::absolute_path(datasetsimpl::join(root, "caltech101")); this->root = root; - TORCH_CHECK( - datasetsimpl::mkpath(root), - "Failed to create directory \"", - root, - "\" for Caltech101 dataset"); - TORCH_CHECK(checkIntegrity(), "Caltech101 dataset not found or corrupted.") categories = diff --git a/torchvision/csrc/datasets/datasetsimpl.cpp b/torchvision/csrc/datasets/datasetsimpl.cpp index 65bef70a017..2f45296fa40 100644 --- a/torchvision/csrc/datasets/datasetsimpl.cpp +++ b/torchvision/csrc/datasets/datasetsimpl.cpp @@ -1,10 +1,27 @@ #include "datasetsimpl.h" +#include +#include +#include +#include + std::vector vision::datasets::datasetsimpl::lsdir( const std::string& path) { std::vector list; - for (const auto& ent : std::filesystem::directory_iterator(path)) - list.push_back(ent.path().filename()); + DIR* dp; + struct dirent* ep; + + dp = opendir(path.c_str()); + if (dp != nullptr) { + while ((ep = readdir(dp))) { + std::string name = ep->d_name; + if (name != "." && name != "..") + list.emplace_back(std::move(name)); + } + + (void)closedir(dp); + } + return list; } @@ -22,21 +39,22 @@ void vision::datasets::datasetsimpl::sort_names( } bool vision::datasets::datasetsimpl::isdir(const std::string& path) { - return std::filesystem::is_directory(path); + struct stat st; + if (stat(path.c_str(), &st) == 0) + return st.st_mode & S_IFDIR; + return false; } bool vision::datasets::datasetsimpl::isfile(const std::string& path) { - return std::filesystem::is_regular_file(path); + struct stat st; + if (stat(path.c_str(), &st) == 0) + return st.st_mode & S_IFREG; + return false; } bool vision::datasets::datasetsimpl::exists(const std::string& path) { - return std::filesystem::exists(path); -} - -bool vision::datasets::datasetsimpl::mkpath(const std::string& path) { - if (exists(path)) - return true; - return std::filesystem::create_directories(path); + struct stat st; + return stat(path.c_str(), &st) == 0; } torch::Tensor vision::datasets::datasetsimpl::read_image( @@ -45,8 +63,6 @@ torch::Tensor vision::datasets::datasetsimpl::read_image( auto mat = cv::imread(path); TORCH_CHECK(!mat.empty(), "Failed to read image \"", path, "\"."); - // TODO make channels variable - mat = transform(mat); std::vector tensors; std::vector channels(size_t(mat.channels())); @@ -63,7 +79,9 @@ torch::Tensor vision::datasets::datasetsimpl::read_image( std::string vision::datasets::datasetsimpl::absolute_path( const std::string& path) { - return std::filesystem::absolute(path).string(); + char rpath[PATH_MAX]; + realpath(path.c_str(), rpath); + return std::string(rpath); } std::function vision::datasets::datasetsimpl:: diff --git a/torchvision/csrc/datasets/datasetsimpl.h b/torchvision/csrc/datasets/datasetsimpl.h index 5f72b1c8e3f..ea3df0a3ede 100644 --- a/torchvision/csrc/datasets/datasetsimpl.h +++ b/torchvision/csrc/datasets/datasetsimpl.h @@ -3,7 +3,6 @@ #include #include -#include #ifndef TORCH_CHECK #define TORCH_CHECK AT_CHECK @@ -25,8 +24,6 @@ bool isfile(const std::string& path); bool exists(const std::string& path); -bool mkpath(const std::string& path); - std::string absolute_path(const std::string& path); inline std::string join(const std::string& str) { @@ -34,7 +31,7 @@ inline std::string join(const std::string& str) { } template inline std::string join(const std::string& head, Tail&&... tail) { - return std::filesystem::path(head).append(join(tail...)).string(); + return head + "/" + join(std::forward(tail)...); } torch::Tensor read_image( @@ -46,8 +43,8 @@ std::function make_transform( int height, cv::ColorConversionCodes code); -inline auto rgb_transform = make_transform(224, 224, cv::COLOR_BGR2RGB); -inline auto gray_transform = make_transform(224, 224, cv::COLOR_BGR2GRAY); +static auto rgb_transform = make_transform(224, 224, cv::COLOR_BGR2RGB); +static auto gray_transform = make_transform(224, 224, cv::COLOR_BGR2GRAY); } // namespace datasetsimpl } // namespace datasets From aee40afd9d40facc1fcd3cfa309b010f9f47de44 Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 29 Jul 2019 16:34:57 +0430 Subject: [PATCH 5/6] Removed Caltech and cv transform --- CMakeLists.txt | 2 +- torchvision/csrc/datasets/caltech.cpp | 58 ---------------------- torchvision/csrc/datasets/caltech.h | 35 ------------- torchvision/csrc/datasets/datasets.h | 16 ------ torchvision/csrc/datasets/datasetsimpl.cpp | 23 +++------ torchvision/csrc/datasets/datasetsimpl.h | 13 +---- 6 files changed, 9 insertions(+), 138 deletions(-) delete mode 100644 torchvision/csrc/datasets/caltech.cpp delete mode 100644 torchvision/csrc/datasets/caltech.h delete mode 100644 torchvision/csrc/datasets/datasets.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f0f116eeb16..8e227a23d8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,5 +24,5 @@ target_link_libraries(convertmodels "${TORCH_LIBRARIES}") install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}) install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/models) -install(FILES ${DATASETS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/datasets) +#install(FILES ${DATASETS_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}/datasets) diff --git a/torchvision/csrc/datasets/caltech.cpp b/torchvision/csrc/datasets/caltech.cpp deleted file mode 100644 index b9fbb428449..00000000000 --- a/torchvision/csrc/datasets/caltech.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "caltech.h" - -namespace vision { -namespace datasets { - -Caltech101::Caltech101( - std::string root, - std::function transform) - : cv_transform(transform) { - root = datasetsimpl::absolute_path(datasetsimpl::join(root, "caltech101")); - this->root = root; - - TORCH_CHECK(checkIntegrity(), "Caltech101 dataset not found or corrupted.") - - categories = - datasetsimpl::lsdir(datasetsimpl::join(root, "101_ObjectCategories")); - datasetsimpl::sort_names(categories); - - auto it = - std::find(categories.begin(), categories.end(), "BACKGROUND_Google"); - if (it != categories.end()) - categories.erase(it); - - long count = 0; - for (const auto& category : categories) { - auto files = datasetsimpl::lsdir( - datasetsimpl::join(root, "101_ObjectCategories", category)); - - for (auto& file : files) - data.emplace_back(std::make_pair(std::move(file), count)); - - ++count; - } -} - -Caltech101::Example Caltech101::get(size_t index) { - auto& pair = data[index]; - auto path = datasetsimpl::join( - root, - "101_ObjectCategories", - categories[size_t(pair.second)], - pair.first); - - auto data = datasetsimpl::read_image(path, cv_transform); - auto target = torch::from_blob(&pair.second, {1}, torch::kLong).clone(); - return {data, target}; -} - -torch::optional Caltech101::size() const { - return data.size(); -} - -bool Caltech101::checkIntegrity() { - return datasetsimpl::exists(datasetsimpl::join(root, "101_ObjectCategories")); -} - -} // namespace datasets -} // namespace vision diff --git a/torchvision/csrc/datasets/caltech.h b/torchvision/csrc/datasets/caltech.h deleted file mode 100644 index b53d489bb36..00000000000 --- a/torchvision/csrc/datasets/caltech.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef CALTECH_H -#define CALTECH_H - -#include -#include "../general.h" -#include "datasetsimpl.h" - -namespace vision { -namespace datasets { - -class VISION_API Caltech101 : torch::data::Dataset { - using Example = torch::data::Example<>; - - std::string root; - std::function cv_transform; - std::vector categories; - std::vector> data; - - bool checkIntegrity(); - - public: - Caltech101( - std::string root, - std::function cv_transform = - datasetsimpl::rgb_transform); - - Example get(size_t index) override; - - torch::optional size() const override; -}; - -} // namespace datasets -} // namespace vision - -#endif // CALTECH_H diff --git a/torchvision/csrc/datasets/datasets.h b/torchvision/csrc/datasets/datasets.h deleted file mode 100644 index b2355b2664c..00000000000 --- a/torchvision/csrc/datasets/datasets.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef DATASETS_H -#define DATASETS_H - -#include "caltech.h" - -namespace vision { -namespace datasets { -namespace cv_transforms { -using datasetsimpl::gray_transform; -using datasetsimpl::make_transform; -using datasetsimpl::rgb_transform; -} // namespace cv_transforms -} // namespace datasets -} // namespace vision - -#endif // DATASETS_H diff --git a/torchvision/csrc/datasets/datasetsimpl.cpp b/torchvision/csrc/datasets/datasetsimpl.cpp index 2f45296fa40..6975a77f580 100644 --- a/torchvision/csrc/datasets/datasetsimpl.cpp +++ b/torchvision/csrc/datasets/datasetsimpl.cpp @@ -1,6 +1,7 @@ #include "datasetsimpl.h" #include +#include #include #include #include @@ -58,12 +59,11 @@ bool vision::datasets::datasetsimpl::exists(const std::string& path) { } torch::Tensor vision::datasets::datasetsimpl::read_image( - const std::string& path, - std::function transform) { + const std::string& path) { auto mat = cv::imread(path); TORCH_CHECK(!mat.empty(), "Failed to read image \"", path, "\"."); - mat = transform(mat); + cv::cvtColor(mat, mat, cv::COLOR_BGR2RGB); std::vector tensors; std::vector channels(size_t(mat.channels())); cv::split(mat, channels); @@ -72,9 +72,10 @@ torch::Tensor vision::datasets::datasetsimpl::read_image( tensors.push_back( torch::from_blob(channel.ptr(), {mat.rows, mat.cols}, torch::kUInt8)); - return torch::cat(tensors) - .view({mat.channels(), mat.rows, mat.cols}) - .to(torch::kFloat); + auto output = torch::cat(tensors) + .view({mat.channels(), mat.rows, mat.cols}) + .to(torch::kFloat); + return output / 255; } std::string vision::datasets::datasetsimpl::absolute_path( @@ -83,13 +84,3 @@ std::string vision::datasets::datasetsimpl::absolute_path( realpath(path.c_str(), rpath); return std::string(rpath); } - -std::function vision::datasets::datasetsimpl:: - make_transform(int width, int height, cv::ColorConversionCodes code) { - return [width, height, code](const cv::Mat& mat) { - cv::Mat new_mat; - cv::resize(mat, new_mat, cv::Size(width, height)); - cv::cvtColor(new_mat, new_mat, code); - return new_mat; - }; -} diff --git a/torchvision/csrc/datasets/datasetsimpl.h b/torchvision/csrc/datasets/datasetsimpl.h index ea3df0a3ede..9b446a81574 100644 --- a/torchvision/csrc/datasets/datasetsimpl.h +++ b/torchvision/csrc/datasets/datasetsimpl.h @@ -1,7 +1,6 @@ #ifndef DATASETSIMPL_H #define DATASETSIMPL_H -#include #include #ifndef TORCH_CHECK @@ -34,17 +33,7 @@ inline std::string join(const std::string& head, Tail&&... tail) { return head + "/" + join(std::forward(tail)...); } -torch::Tensor read_image( - const std::string& path, - std::function transform); - -std::function make_transform( - int width, - int height, - cv::ColorConversionCodes code); - -static auto rgb_transform = make_transform(224, 224, cv::COLOR_BGR2RGB); -static auto gray_transform = make_transform(224, 224, cv::COLOR_BGR2GRAY); +torch::Tensor read_image(const std::string& path); } // namespace datasetsimpl } // namespace datasets From adc716de77b688c1b8dea2c405e69b45fe7615ab Mon Sep 17 00:00:00 2001 From: ShahriarSS Date: Mon, 29 Jul 2019 16:56:51 +0430 Subject: [PATCH 6/6] Fixed a few stuff --- torchvision/csrc/datasets/datasetsimpl.cpp | 44 ++++++++++++---------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/torchvision/csrc/datasets/datasetsimpl.cpp b/torchvision/csrc/datasets/datasetsimpl.cpp index 6975a77f580..795b5e2dd25 100644 --- a/torchvision/csrc/datasets/datasetsimpl.cpp +++ b/torchvision/csrc/datasets/datasetsimpl.cpp @@ -6,60 +6,61 @@ #include #include -std::vector vision::datasets::datasetsimpl::lsdir( - const std::string& path) { +namespace vision { +namespace datasets { +namespace datasetsimpl { +std::vector lsdir(const std::string& path) { std::vector list; - DIR* dp; - struct dirent* ep; + auto dp = opendir(path.c_str()); - dp = opendir(path.c_str()); if (dp != nullptr) { - while ((ep = readdir(dp))) { + auto ep = readdir(dp); + + while (ep != nullptr) { std::string name = ep->d_name; if (name != "." && name != "..") list.emplace_back(std::move(name)); } - (void)closedir(dp); + closedir(dp); } return list; } -std::string vision::datasets::datasetsimpl::tolower(std::string str) { +std::string tolower(std::string str) { std::transform(str.begin(), str.end(), str.begin(), ::tolower); return str; } -void vision::datasets::datasetsimpl::sort_names( - std::vector& data) { - auto comp = [](const std::string& A, const std::string& B) { - return tolower(A) < tolower(B); - }; +inline bool comp(const std::string& A, const std::string& B) { + return tolower(A) < tolower(B); +}; + +void sort_names(std::vector& data) { std::sort(data.begin(), data.end(), comp); } -bool vision::datasets::datasetsimpl::isdir(const std::string& path) { +bool isdir(const std::string& path) { struct stat st; if (stat(path.c_str(), &st) == 0) return st.st_mode & S_IFDIR; return false; } -bool vision::datasets::datasetsimpl::isfile(const std::string& path) { +bool isfile(const std::string& path) { struct stat st; if (stat(path.c_str(), &st) == 0) return st.st_mode & S_IFREG; return false; } -bool vision::datasets::datasetsimpl::exists(const std::string& path) { +bool exists(const std::string& path) { struct stat st; return stat(path.c_str(), &st) == 0; } -torch::Tensor vision::datasets::datasetsimpl::read_image( - const std::string& path) { +torch::Tensor read_image(const std::string& path) { auto mat = cv::imread(path); TORCH_CHECK(!mat.empty(), "Failed to read image \"", path, "\"."); @@ -78,9 +79,12 @@ torch::Tensor vision::datasets::datasetsimpl::read_image( return output / 255; } -std::string vision::datasets::datasetsimpl::absolute_path( - const std::string& path) { +std::string absolute_path(const std::string& path) { char rpath[PATH_MAX]; realpath(path.c_str(), rpath); return std::string(rpath); } + +} // namespace datasetsimpl +} // namespace datasets +} // namespace vision