diff --git a/src/shogun/base/macros.h b/src/shogun/base/macros.h index 0f27458c0b3..61c2912ca98 100644 --- a/src/shogun/base/macros.h +++ b/src/shogun/base/macros.h @@ -10,12 +10,15 @@ #if defined(__GNUC__) || defined(__APPLE__) #define SG_FORCED_INLINE inline __attribute__((always_inline)) #define SG_FORCED_PACKED __attribute__((__packed__)) +#define SG_ATTRIBUTE_UNUSED __attribute__((unused)) #elif defined(_MSC_VER) #define SG_FORCED_INLINE __forceinline #define SG_FORCED_PACKED +#define SG_ATTRIBUTE_UNUSED #else #define SG_FORCED_INLINE #define SG_FORCED_PACKED +#define SG_ATTRIBUTE_UNUSED #endif // a quick macro for making sure that an object diff --git a/src/shogun/io/fs/FileSystem.h b/src/shogun/io/fs/FileSystem.h new file mode 100644 index 00000000000..03cb65f2b7e --- /dev/null +++ b/src/shogun/io/fs/FileSystem.h @@ -0,0 +1,193 @@ +/* + * This software is distributed under BSD 3-clause license (see LICENSE file). + * + * Authors: Viktor Gal + */ + +#ifndef SHOGUN_FILESYSTEM_H +#define SHOGUN_FILESYSTEM_H + +#include +#include +#include +#include + +#include +#include +#include + +namespace shogun +{ +class RandomAccessFile; +class WritableFile; + +/** + * Interface representing a filesystem. + * A unified interface to open read and write files. + */ +class FileSystem +{ +public: + FileSystem() {} + + virtual ~FileSystem() {} + + /** + * Create a new random access read-only file + * + * @param name file name string + * @return unique pointer to the file or exception in case of error + */ + virtual std::unique_ptr new_random_access_file(const std::string& fname) = 0; + + /** + * Create a new writable file + * + * @param name file name string + * @return unique pointer to the file or exception in case of error + */ + virtual std::unique_ptr new_writable_file(const std::string& fname) = 0; + + /** + * Create a new writable file starting on the end of the file + * + * @param name file name string + * @return unique pointer to the file or exception in case of error + */ + virtual std::unique_ptr new_appendable_file(const std::string& fname) = 0; + + /** + * Check if file exists + * + * @return True in case the file exists False otherwise. + */ + virtual bool file_exists(const std::string& fname) = 0; + + /** + * Delete a given file + * + * @param fname file name to be deleted + */ + virtual void delete_file(const std::string& fname) = 0; + + /** + * Create directory + * + * @param dirname name of the directory to create + */ + virtual void create_dir(const std::string& dirname) = 0; + + /** + * Delete directory + * + * @param dirname name of the directory to delete + */ + virtual void delete_dir(const std::string& dirname) = 0; + + /** + * Rename file + * + * @param src file name to rename + * @param target of the renaming + */ + virtual void rename_file(const std::string& src, const std::string& target) = 0; + + /** + * Translate name. + * Resolves and cleans up the URI + * + * @param name the URI of the file + */ + virtual std::string translate_name(const std::string& name) const + { + // TODO: clean the path from junk + return name; + } + + /** + * Checks whether a given path is a directory or not. + * + * @param name path to the directory + */ + virtual bool is_directory(const std::string& fname) = 0; + + virtual uint64_t get_file_size(const std::string& fname) = 0; +}; + +/** + * A file abstraction for randomly reading the contents of a file. + */ +class RandomAccessFile +{ +public: + RandomAccessFile() {} + virtual ~RandomAccessFile() {} + + virtual void read( + uint64_t offset, size_t n, + Chunk* result, char* scratch) const = 0; +private: + SG_DELETE_COPY_AND_ASSIGN(RandomAccessFile); +}; + +/** + * A file abstraction for sequentially writing out a file. + */ +class WritableFile +{ +public: + WritableFile() {} + virtual ~WritableFile() {} + + /** + * append data to file + */ + virtual void append(const void* data, size_t size) = 0; + + /** + * Close the file + */ + virtual void close() = 0; + + /** + * Flush the file + */ + virtual void flush() = 0; + + /** + * Sync the content of the file to the file system + */ + virtual void sync() = 0; + +private: + SG_DELETE_COPY_AND_ASSIGN(WritableFile); +}; + +namespace internal { + +template +struct FileSystemRegister +{ + FileSystemRegister(FileSystemRegistry* fsr, const std::string& scheme) + { + fsr->register_fs(scheme, []() -> FileSystem* { return new Factory; }); + } +}; + +} // namespace internal + +#define REGISTER_FILE_SYSTEM_FACTORY(fsr, scheme, factory) \ + REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, fsr, scheme, factory) +#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, fsr, scheme, factory) \ + REGISTER_FILE_SYSTEM_UNIQ(ctr, fsr, scheme, factory) +#define REGISTER_FILE_SYSTEM_UNIQ(ctr, fsr, scheme, factory) \ + static ::shogun::internal::FileSystemRegister \ + register_ff##ctr SG_ATTRIBUTE_UNUSED = \ + ::shogun::internal::FileSystemRegister(fsr, scheme); + +#define REGISTER_FILE_SYSTEM(scheme, factory) \ + REGISTER_FILE_SYSTEM_FACTORY(FileSystemRegistry::instance(), scheme, factory); + +} // namespace shogun + +#endif diff --git a/src/shogun/io/fs/FileSystemRegistry.cpp b/src/shogun/io/fs/FileSystemRegistry.cpp new file mode 100644 index 00000000000..8f0c2a6c336 --- /dev/null +++ b/src/shogun/io/fs/FileSystemRegistry.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +using namespace shogun; +using namespace std; + +void FileSystemRegistry::register_fs( + const string& scheme, + FileSystemRegistry::Factory factory) +{ + lock_guard lock(m_mutex); + if (!m_fs_registry.emplace( + scheme, unique_ptr(factory())).second) + { + throw ShogunException("File factor for " + scheme + " already registered"); + } +} + +FileSystem* FileSystemRegistry::lookup(const string& scheme) const +{ + lock_guard lock(m_mutex); + const auto found = m_fs_registry.find(scheme); + return (found == m_fs_registry.end()) + ? nullptr + : found->second.get(); +} + +FileSystem* FileSystemRegistry::get_file_system_for_file( + const string& fname) const +{ + string scheme, host, path; + //io::ParseURI(fname, &scheme, &host, &path); + FileSystem* fs = lookup(scheme); + if (!fs) + { + if (scheme.empty()) + scheme = "[local]"; + throw ShogunException( + "File system scheme '" + scheme + + "' not implemented (file: '" + scheme + "')"); + } + return fs; +} + + +vector FileSystemRegistry::get_registered_file_system_schemes() +{ + vector schemes; + lock_guard lock(m_mutex); + for (const auto& e: m_fs_registry) + schemes.push_back(e.first); + return schemes; +} + +unique_ptr FileSystemRegistry::new_random_access_file(const string& fname) +{ + return get_file_system_for_file(fname)->new_random_access_file(fname); +} + + +unique_ptr FileSystemRegistry::new_writable_file(const string& fname) +{ + return get_file_system_for_file(fname)->new_writable_file(fname); +} + +unique_ptr FileSystemRegistry::new_appendable_file(const string& fname) +{ + return get_file_system_for_file(fname)->new_appendable_file(fname); +} + +bool FileSystemRegistry::file_exists(const string& fname) +{ + return get_file_system_for_file(fname)->file_exists(fname); +} + +void FileSystemRegistry::delete_file(const string& fname) +{ + return get_file_system_for_file(fname)->delete_file(fname); +} + +void FileSystemRegistry::create_dir(const string& dirname) +{ + return get_file_system_for_file(dirname)->create_dir(dirname); +} + +void FileSystemRegistry::delete_dir(const string& dirname) +{ + return get_file_system_for_file(dirname)->delete_dir(dirname); +} + +void FileSystemRegistry::rename_file(const string& src, const string& target) +{ + FileSystem* src_fs = get_file_system_for_file(src); + FileSystem* target_fs = get_file_system_for_file(src); + if (src_fs != target_fs) + throw ShogunException( + "Renaming "+ src +" to "+ target +" not implemented"); + + return src_fs->rename_file(src, target); +} + +string FileSystemRegistry::translate_name(const string& name) const +{ + return get_file_system_for_file(name)->translate_name(name); +} + +bool FileSystemRegistry::is_directory(const string& fname) +{ + return get_file_system_for_file(fname)->is_directory(fname); +} + +uint64_t FileSystemRegistry::get_file_size(const string& fname) +{ + return get_file_system_for_file(fname)->get_file_size(fname); +} + diff --git a/src/shogun/io/fs/FileSystemRegistry.h b/src/shogun/io/fs/FileSystemRegistry.h new file mode 100644 index 00000000000..5da64b5a1f6 --- /dev/null +++ b/src/shogun/io/fs/FileSystemRegistry.h @@ -0,0 +1,82 @@ +/* + * This software is distributed under BSD 3-clause license (see LICENSE file). + * + * Authors: Viktor Gal + */ + +#ifndef SHOGUN_FILESYSTEM_REGISTRY_H +#define SHOGUN_FILESYSTEM_REGISTRY_H + +#include +#include + +namespace shogun +{ + +class FileSystem; +class RandomAccessFile; +class WritableFile; + +class FileSystemRegistry +{ +public: + typedef std::function Factory; + + + static FileSystemRegistry* instance() { + static FileSystemRegistry* fsr = new FileSystemRegistry(); + return fsr; + } + + /** + * + */ + void register_fs(const std::string& scheme, Factory factory); + + /** + * + */ + FileSystem* lookup(const std::string& scheme) const; + + + /** + * + */ + FileSystem* get_file_system_for_file(const std::string& fname) const; + + /** + * + */ + std::vector get_registered_file_system_schemes(); + + std::unique_ptr new_random_access_file(const std::string& fname); + + std::unique_ptr new_writable_file(const std::string& fname); + + std::unique_ptr new_appendable_file(const std::string& fname); + + bool file_exists(const std::string& fname); + + void delete_file(const std::string& fname); + + void create_dir(const std::string& dirname); + + void delete_dir(const std::string& dirname); + + void rename_file(const std::string& src, const std::string& target); + + std::string translate_name(const std::string& name) const; + bool is_directory(const std::string& fname); + + uint64_t get_file_size(const std::string& fname); + +private: + FileSystemRegistry() {} + + mutable std::mutex m_mutex; + std::unordered_map> m_fs_registry; +}; + +} + +#endif /* SHOGUN_FILESYSTEM_REGISTRY_H */ diff --git a/src/shogun/io/fs/NullFileSystem.h b/src/shogun/io/fs/NullFileSystem.h new file mode 100644 index 00000000000..1d6acb93a9b --- /dev/null +++ b/src/shogun/io/fs/NullFileSystem.h @@ -0,0 +1,69 @@ +#ifndef __NULL_FILE_SYSTEM_H__ +#define __NULL_FILE_SYSTEM_H__ + +#include +#include + +namespace shogun +{ +class NullFileSystem : public FileSystem +{ +public: + NullFileSystem() {} + + ~NullFileSystem() override = default; + + std::unique_ptr new_random_access_file( + const std::string& fname) override + { + throw ShogunNotImplementedException("new_random_access_file unimplemented"); + } + + std::unique_ptr new_writable_file( + const std::string& fname) override + { + throw ShogunNotImplementedException("NewWritableFile new_writable_file"); + } + + std::unique_ptr new_appendable_file( + const std::string& fname) override + { + throw ShogunNotImplementedException("new_appendable_file unimplemented"); + } + + bool file_exists(const std::string& fname) override + { + throw ShogunNotImplementedException("file_exists unimplemented"); + } + + void delete_file(const std::string& fname) override + { + throw ShogunNotImplementedException("delete_file unimplemented"); + } + + void create_dir(const std::string& dirname) override + { + throw ShogunNotImplementedException("create_dir unimplemented"); + } + + void delete_dir(const std::string& dirname) override + { + throw ShogunNotImplementedException("delete_dir unimplemented"); + } + + uint64_t get_file_size(const std::string& fname) override + { + throw ShogunNotImplementedException("get_file_size unimplemented"); + } + + void rename_file(const std::string& src, const std::string& target) override + { + throw ShogunNotImplementedException("rename_file unimplemented"); + } + +}; + +} + +#endif + diff --git a/src/shogun/io/fs/PosixFileSystem.cpp b/src/shogun/io/fs/PosixFileSystem.cpp new file mode 100644 index 00000000000..5ce1d484b5a --- /dev/null +++ b/src/shogun/io/fs/PosixFileSystem.cpp @@ -0,0 +1,203 @@ +#ifndef _MSC_VER + +#include +#include +#include +#include +#include +#include +#if !defined(__APPLE__) +#include +#endif +#include +#include +#include +#include +#include +#include + +#include + +using namespace shogun; +using namespace std; + +class PosixRandomAccessFile: public RandomAccessFile +{ +public: + PosixRandomAccessFile(const string& fname, int fd): + m_filename(fname), + m_fd(fd) + { + } + + ~PosixRandomAccessFile() override + { + close(m_fd); + } + + void read(uint64_t offset, size_t n, Chunk* result, char* scratch) const override + { + char* dst = scratch; + while (n > 0) + { + ssize_t r = pread(m_fd, dst, n, static_cast(offset)); + if (r > 0) + { + dst += r; + n -= r; + offset += r; + } + else if (r == 0) + { + throw system_error(make_error_code(errc::result_out_of_range), "Read less bytes than requested"); + } + else if (errno == EINTR || errno == EAGAIN) + { + // should retry... + } + else + { + throw system_error(errno, generic_category()); + } + } + *result = Chunk(scratch, dst - scratch); + } + +private: + string m_filename; + int m_fd; +}; + +class PosixWritableFile: public WritableFile +{ +public: + PosixWritableFile(const string& fname, FILE* f): + m_filename(fname), + m_file(f) + { + } + + ~PosixWritableFile() override + { + if (m_file != nullptr) + { + // Ignoring any potential errors + fclose(m_file); + } + } + + void append(const void* data, size_t size) override + { + size_t r = fwrite(data, 1, size, m_file); + if (r != size) + throw system_error(errno, generic_category()); + } + + void close() override + { + if (fclose(m_file) != 0) + throw system_error(errno, generic_category()); + m_file = nullptr; + } + + void flush() override + { + if (fflush(m_file) != 0) + throw system_error(errno, generic_category()); + } + + void sync() override + { + if (fflush(m_file) != 0) + throw system_error(errno, generic_category()); + } + +private: + string m_filename; + FILE* m_file; +}; + +unique_ptr PosixFileSystem::new_random_access_file( + const string& fname) +{ + string translated_fname = translate_name(fname); + int fd = open(translated_fname.c_str(), O_RDONLY); + if (fd < 0) + throw system_error(errno, generic_category(), "could not create random access"); + return make_unique(translated_fname, fd); +} + +unique_ptr PosixFileSystem::new_writable_file(const string& fname) +{ + string translated_fname = translate_name(fname); + FILE* f = fopen(translated_fname.c_str(), "w"); + if (f == nullptr) + throw system_error(errno, generic_category(), "could not create writable file"); + return make_unique(translated_fname, f); +} + +unique_ptr PosixFileSystem::new_appendable_file(const string& fname) +{ + string translated_fname = translate_name(fname); + FILE* f = fopen(translated_fname.c_str(), "a"); + if (f == nullptr) + throw system_error(errno, generic_category(), "could not create file"); + return make_unique(translated_fname, f); +} + +bool PosixFileSystem::file_exists(const string& fname) +{ + return (access(translate_name(fname).c_str(), F_OK) == 0) + ? true : false; +} + +void PosixFileSystem::delete_file(const string& fname) +{ + if (unlink(translate_name(fname).c_str()) != 0) + throw system_error(errno, generic_category()); +} + +void PosixFileSystem::create_dir(const string& name) +{ + if (mkdir(translate_name(name).c_str(), 0755) != 0) + throw system_error(errno, generic_category()); +} + +void PosixFileSystem::delete_dir(const string& name) +{ + if (rmdir(translate_name(name).c_str()) != 0) + throw system_error(errno, generic_category()); +} + + +void PosixFileSystem::rename_file(const string& src, const string& target) +{ + if (rename(translate_name(src).c_str(), translate_name(target).c_str()) != 0) + throw system_error(errno, generic_category()); +} + +bool PosixFileSystem::is_directory(const std::string& fname) +{ + if (!file_exists(fname)) + return false; + + struct stat sbuf; + if (stat(translate_name(fname).c_str(), &sbuf) != 0) + return false; + + return S_ISDIR(sbuf.st_mode); +} + +uint64_t PosixFileSystem::get_file_size(const std::string& fname) +{ + struct stat sbuf; + if (stat(translate_name(fname).c_str(), &sbuf) != 0) + throw system_error(errno, generic_category()); + + return sbuf.st_size; +} + +REGISTER_FILE_SYSTEM("", PosixFileSystem); +REGISTER_FILE_SYSTEM("file", LocalPosixFileSystem); + +#endif diff --git a/src/shogun/io/fs/PosixFileSystem.h b/src/shogun/io/fs/PosixFileSystem.h new file mode 100644 index 00000000000..63c8a1e97da --- /dev/null +++ b/src/shogun/io/fs/PosixFileSystem.h @@ -0,0 +1,54 @@ +#ifndef __POSIX_FILE_SYSTEM_H__ +#define __POSIX_FILE_SYSTEM_H__ +#ifndef _MSC_VER + +#include + +namespace shogun +{ + +class PosixFileSystem : public FileSystem +{ +public: + PosixFileSystem() {} + + ~PosixFileSystem() {} + + std::unique_ptr new_random_access_file( + const std::string& filename) override; + + std::unique_ptr new_writable_file( + const std::string& fname) override; + + std::unique_ptr new_appendable_file( + const std::string& fname) override; + + bool file_exists(const std::string& fname) override; + + void delete_file(const std::string& fname) override; + + void create_dir(const std::string& name) override; + + void delete_dir(const std::string& name) override; + + void rename_file(const std::string& src, const std::string& target) override; + + uint64_t get_file_size(const std::string& fname) override; + + bool is_directory(const std::string& fname) override; +}; + +class LocalPosixFileSystem : public PosixFileSystem +{ +public: + std::string translate_name(const std::string& name) const override + { + Chunk scheme, host, path; + return path.to_string(); + } +}; + +} // namespace shogun + +#endif +#endif // __POSIX_FILE_SYSTEM_H__ diff --git a/src/shogun/io/fs/WindowsFileSystem.cpp b/src/shogun/io/fs/WindowsFileSystem.cpp new file mode 100644 index 00000000000..ce71f5e6f27 --- /dev/null +++ b/src/shogun/io/fs/WindowsFileSystem.cpp @@ -0,0 +1,184 @@ +#ifdef _MSC_VER +#include +#include +#include +#include +#include +#include +#undef StrCat +#include +#include +#include +#include + +#include + +#include + +using namespace shogun; +using namespace std; + +class WindowsRandomAccessFile: public RandomAccessFile +{ +public: + WindowsRandomAccessFile(const string& fname, HANDLE hfile): + m_filename(fname), m_hfile(hfile) {} + + ~WindowsRandomAccessFile() override + { + if (m_hfile != NULL && m_hfile != INVALID_HANDLE_VALUE) + ::CloseHandle(m_hfile); + } + + void read(uint64_t offset, size_t n, Chunk* result, char* scratch) const override + { + char* dst = scratch; + while (n > 0) { + SSIZE_T r = pread(m_hfile, dst, n, offset); + if (r > 0) + { + offset += r; + dst += r; + n -= r; + } + else if (r == 0) + { + throw system_error( + make_error_code(errc::result_out_of_range), + "Read less bytes than requested"); + + } + else if (errno == EINTR || errno == EAGAIN) { + // Retry + } + else + { + throw system_error(::GetLastError(), system_category()); + } + } + *result = Chunk(scratch, dst - scratch); + } + +private: + string m_filename; + HANDLE m_hfile; +}; + +class WindowsWritableFile: public WritableFile +{ +public: + WindowsWritableFile(const string& fname, HANDLE hfile): + m_filename(fname), m_hfile(hfile) {} + + ~WindowsWritableFile() override + { + if (m_hfile != NULL && m_hfile != INVALID_HANDLE_VALUE) + WindowsWritableFile::close(); + } + + void append(const Chunk& data) override + { + DWORD bytes_written = 0; + DWORD data_size = static_cast(data.size()); + BOOL write_result = + ::WriteFile(m_hfile, data.data(), data_size, &bytes_written, NULL); + if (FALSE == write_result) + throw system_error(::GetLastError(), system_category()); + + assert(size_t(bytes_written) == data.size()); + } + + void close() override + { + assert(INVALID_HANDLE_VALUE != m_hfile); + + Status result = Flush(); + if (!result.ok()) { + return result; + } + + if (FALSE == ::CloseHandle(m_hfile)) + throw system_error(::GetLastError(), system_category()); + + m_hfile = INVALID_HANDLE_VALUE; + } + + void flush() override + { + if (FALSE == ::FlushFileBuffers(m_hfile)) + throw system_error(::GetLastError(), system_category()); + } + + void sync() override + { + flush(); + } + +private: + string m_filename; + HANDLE* m_hfile; +}; + +unique_ptr WindowsFileSystem::new_random_access_file( + const string& fname) +{ + string translated_fname = translate_name(fname); + int fd = open(translated_fname.c_str(), O_RDONLY); + if (fd < 0) + throw system_error(errno, generic_category(), "could not create random access"); + return make_unique(translated_fname, fd); +} + +unique_ptr WindowsFileSystem::new_writable_file(const string& fname) +{ + string translated_fname = translate_name(fname); + FILE* f = fopen(translated_fname.c_str(), "w"); + if (f == nullptr) + throw system_error(errno, generic_category(), "could not create writable file"); + return make_unique(translated_fname, f); +} + +unique_ptr WindowsFileSystem::new_appendable_file(const string& fname) +{ + string translated_fname = translate_name(fname); + FILE* f = fopen(translated_fname.c_str(), "a"); + if (f == nullptr) + throw system_error(errno, generic_category(), "could not create file"); + return make_unique(translated_fname, f); +} + +bool WindowsFileSystem::file_exists(const string& fname) +{ + return (access(translate_name(fname).c_str(), F_OK) == 0) + ? true : false; +} + +void WindowsFileSystem::delete_file(const string& fname) +{ + if (unlink(translate_name(fname).c_str()) != 0) + throw system_error(errno, generic_category()); +} + +void WindowsFileSystem::create_dir(const string& name) +{ + if (mkdir(translate_name(name).c_str(), 0755) != 0) + throw system_error(errno, generic_category()); +} + +void WindowsFileSystem::delete_dir(const string& name) +{ + if (rmdir(translate_name(name).c_str()) != 0) + throw system_error(errno, generic_category()); +} + + +void WindowsFileSystem::rename_file(const string& src, const string& target) +{ + if (rename(translate_name(src).c_str(), translate_name(target).c_str()) != 0) + throw system_error(errno, generic_category()); +} + +REGISTER_FILE_SYSTEM("", WindowsFileSystem); +REGISTER_FILE_SYSTEM("file", LocalWindowsFileSystem); + +#endif diff --git a/src/shogun/io/fs/WindowsFileSystem.h b/src/shogun/io/fs/WindowsFileSystem.h new file mode 100644 index 00000000000..5f191704318 --- /dev/null +++ b/src/shogun/io/fs/WindowsFileSystem.h @@ -0,0 +1,53 @@ +#ifndef __WINDOWS_FILE_SYSTEM_H__ +#define __WINDOWS_FILE_SYSTEM_H__ + +#include + +namespace shogun +{ + +class WindowsFileSystem : public FileSystem +{ +public: + WindowsFileSystem() {} + + ~WindowsFileSystem() {} + + std::unique_ptr new_random_access_file( + const std::string& filename) override; + + std::unique_ptr new_writable_file( + const std::string& fname) override; + + std::unique_ptr new_appendable_file( + const std::string& fname) override; + + bool file_exists(const std::string& fname) override; + + void delete_file(const std::string& fname) override; + + void create_dir(const std::string& name) override; + + void delete_dir(const std::string& name) override; + + void rename_file(const std::string& src, const std::string& target) override; + + uint64_t get_file_size(const std::string& fname) override; + + bool is_directory(const std::string& fname) override; +}; + +class LocalWindowsFileSystem : public WindowsFileSystem +{ +public: + std::string translate_name(const std::string& name) const override + { + Chunk scheme, host, path; + // FIXME: parse UIR + return path.to_string(); + } +}; + +} // namespace shogun + +#endif // __WINDOWS_FILE_SYSTEM_H__ diff --git a/src/shogun/io/serialization/JsonSerializer.cpp b/src/shogun/io/serialization/JsonSerializer.cpp index 89d32676111..822a5ac053f 100644 --- a/src/shogun/io/serialization/JsonSerializer.cpp +++ b/src/shogun/io/serialization/JsonSerializer.cpp @@ -9,7 +9,6 @@ using namespace shogun; -// TODO: buffering struct COutputStreamAdapter { typedef char Ch; @@ -20,6 +19,7 @@ struct COutputStreamAdapter void Flush() { + m_stream->flush(); } COutputStream* m_stream; @@ -40,5 +40,6 @@ void CJsonSerializer::write(Some object) writer.StartObject(); writer.Key("name"); writer.String(object->get_name()); + //writer.String(object.get()); writer.EndObject(); } diff --git a/src/shogun/io/stream/BufferedOutputStream.h b/src/shogun/io/stream/BufferedOutputStream.h new file mode 100644 index 00000000000..3a38eae9575 --- /dev/null +++ b/src/shogun/io/stream/BufferedOutputStream.h @@ -0,0 +1,74 @@ +/** This software is distributed under BSD 3-clause license (see LICENSE file). + * + * Authors: Vitktor Gal + */ +#ifndef __BUFFERED_OUTPUT_STREAM_H__ +#define __BUFFERED_OUTPUT_STREAM_H__ + +#include +#include +#include + +namespace shogun +{ + IGNORE_IN_CLASSLIST class CBufferedOutputStream : public COutputStream + { + public: + /** + * Construct a buffered output stream + * + * @param os + * @param buffer_bytes + */ + CBufferedOutputStream(std::shared_ptr os, index_t buffer_bytes = 4096): + COutputStream(), m_os(std::move(os)) + { + + } + + CBufferedOutputStream(CBufferedOutputStream&& src): + COutputStream(), m_os(std::move(src.m_os)) + { + src.m_os = nullptr; + } + + CBufferedOutputStream& operator=(CBufferedOutputStream&& src) + { + m_os = std::move(src.m_os); + return *this; + } + + ~CBufferedOutputStream() override + { + m_os->flush(); + m_os->close(); + } + + void write(void* buffer, size_t size) override + { + m_os->write(buffer, size); + } + + void close() override + { + m_os->close(); + } + + void flush() override + { + m_os->flush(); + } + + const char* get_name() const override + { + return "BufferedOutputStream"; + } + + private: + std::shared_ptr m_os; + + SG_DELETE_COPY_AND_ASSIGN(CBufferedOutputStream); + }; +} + +#endif /* __BUFFERED_OUTPUT_STREAM_H__ */ diff --git a/src/shogun/io/stream/FileOutputStream.h b/src/shogun/io/stream/FileOutputStream.h new file mode 100644 index 00000000000..bbe34838289 --- /dev/null +++ b/src/shogun/io/stream/FileOutputStream.h @@ -0,0 +1,48 @@ +/** This software is distributed under BSD 3-clause license (see LICENSE file). + * + * Authors: Vitktor Gal + */ +#ifndef __FILE_OUTPUT_STREAM_H__ +#define __FILE_OUTPUT_STREAM_H__ + +#include +#include + +namespace shogun +{ +#define IGNORE_IN_CLASSLIST + + IGNORE_IN_CLASSLIST class CFileOutputStream : public COutputStream + { + public: + CFileOutputStream(std::unique_ptr dest): + COutputStream(), m_dest(std::move(dest)) {} + ~CFileOutputStream() override + { + } + + void close() override + { + m_dest->close(); + } + + void flush() override + { + m_dest->flush(); + } + + void write(const void* buffer, size_t size) override + { + m_dest->append(buffer, size); + } + + const char* get_name() const override { return "FileOutputStream"; } + + private: + std::unique_ptr m_dest; + + SG_DELETE_COPY_AND_ASSIGN(CFileOutputStream); + }; +} + +#endif diff --git a/src/shogun/io/stream/OutputStream.h b/src/shogun/io/stream/OutputStream.h index 65acc09e6d1..adc1036dd76 100644 --- a/src/shogun/io/stream/OutputStream.h +++ b/src/shogun/io/stream/OutputStream.h @@ -15,7 +15,9 @@ namespace shogun COutputStream(); virtual ~COutputStream(); - virtual void write(void* buffer, size_t size) = 0; + virtual void close() = 0; + virtual void flush() = 0; + virtual void write(const void* buffer, size_t size) = 0; }; } diff --git a/src/shogun/lib/ShogunNotImplementedException.h b/src/shogun/lib/ShogunNotImplementedException.h new file mode 100644 index 00000000000..a1724ae548b --- /dev/null +++ b/src/shogun/lib/ShogunNotImplementedException.h @@ -0,0 +1,45 @@ +/* + * This software is distributed under BSD 3-clause license (see LICENSE file). + * + * Authors: Viktor Gal + */ + +#ifndef _SHOGUN_NOT_IMPLEMENTED_EXCEPTION_H_ +#define _SHOGUN_NOT_IMPLEMENTED_EXCEPTION_H_ + +#include + +namespace shogun +{ +/** @brief Class ShogunNotImplementedException defines an exception which is thrown whenever an + * error inside of shogun occurs. + */ +class ShogunNotImplementedException: public ShogunException +{ + public: + /** constructor + * + * @param str exception string + */ + explicit ShogunNotImplementedException(const std::string& what_arg) + : ShogunException(what_arg) {} + + /** constructor + * + * @param str exception string + */ + explicit ShogunNotImplementedException(const char* what_arg) + : ShogunException(what_arg) {} + + + /** destructor + */ + virtual ~ShogunNotImplementedException(); + + + private: + /** exception string */ + std::string msg; +}; +} +#endif // _SHOGUN_NOT_IMPLEMENTED_EXCEPTION_H_ diff --git a/src/shogun/lib/chunk.h b/src/shogun/lib/chunk.h new file mode 100644 index 00000000000..86d4fde3821 --- /dev/null +++ b/src/shogun/lib/chunk.h @@ -0,0 +1,45 @@ +#ifndef _CHUNK_H_ +#define _CHUNK_H_ + +#include + +namespace shogun +{ +class Chunk +{ +public: + using size_type = size_t; + + Chunk(): m_data(nullptr), m_size(0) {} + Chunk(const char* d, size_t n) : m_data(d), m_size(n) {} + Chunk(const std::string& s) : m_data(s.data()), m_size(s.size()) {} + Chunk(const char* s) : m_data(s), m_size(strlen(s)) {} + + const char* data() const { return m_data; } + size_t size() const { return m_size; } + + bool empty() const { return m_size == 0; } + + typedef const char* const_iterator; + typedef const char* iterator; + iterator begin() const { return m_data; } + iterator end() const { return m_data + m_size; } + + static const size_t npos = size_type(-1); + + char operator[](size_t n) const + { + assert(n < size()); + return m_data[n]; + } + + std::string to_string() const { return std::string(m_data, m_size); } + +private: + const char* m_data; + size_t m_size; +}; + +} + +#endif /** _CHUNK_H_ **/ \ No newline at end of file diff --git a/src/shogun/lib/exception/ShogunException.h b/src/shogun/lib/exception/ShogunException.h index 26ab9a15ed6..311217a79ae 100644 --- a/src/shogun/lib/exception/ShogunException.h +++ b/src/shogun/lib/exception/ShogunException.h @@ -45,7 +45,7 @@ namespace shogun virtual const char* what() const noexcept override; - private: + protected: /** exception string */ std::string msg; }; diff --git a/tests/unit/io/JsonSerialization_unittest.cc b/tests/unit/io/JsonSerialization_unittest.cc index 7cd1296fd88..6e4313b6a86 100644 --- a/tests/unit/io/JsonSerialization_unittest.cc +++ b/tests/unit/io/JsonSerialization_unittest.cc @@ -14,12 +14,16 @@ class CDummyOutputStream : public COutputStream CDummyOutputStream() : COutputStream(), m_buffer() { } - void write(void* buffer, size_t size) + + void close() override {} + void flush() override {} + + void write(const void* buffer, size_t size) override { std::copy( (char*)buffer, (char*)buffer + size, std::back_inserter(m_buffer)); } - virtual const char* get_name() const + const char* get_name() const override { return "DummyOutputStream"; } diff --git a/tests/unit/io/stream/FileOutputStream_unittest.cc b/tests/unit/io/stream/FileOutputStream_unittest.cc new file mode 100644 index 00000000000..43294f0e1a7 --- /dev/null +++ b/tests/unit/io/stream/FileOutputStream_unittest.cc @@ -0,0 +1,25 @@ +#include + +#include +#include +#include + +using namespace shogun; +using namespace std; + +TEST(FileOutputStream, write) +{ + string fname = "test123"; + std::string test_str("asdf"); + auto fs_registry = FileSystemRegistry::instance(); + auto file = fs_registry->new_writable_file(fname); + auto fos = Some::from_raw(new CFileOutputStream(std::move(file))); + fos->write(test_str.data(), test_str.length()); + fos->close(); + + std::ifstream is(fname); + char str_in[5]; + is.get(&str_in[0], 5); + EXPECT_EQ(test_str, std::string(str_in)); + unlink(fname.c_str()); +}