add filesystem support

src/shogun/base/macros.h

@@ -10,12 +10,15 @@
 #if defined(__GNUC__) || defined(__APPLE__)
 #define SG_FORCED_INLINE inline __attribute__((always_inline))
 #define SG_FORCED_PACKED __attribute__((__packed__))
+#define SG_ATTRIBUTE_UNUSED __attribute__((unused))
 #elif defined(_MSC_VER)
 #define SG_FORCED_INLINE __forceinline
 #define SG_FORCED_PACKED
+#define SG_ATTRIBUTE_UNUSED
 #else
 #define SG_FORCED_INLINE
 #define SG_FORCED_PACKED
+#define SG_ATTRIBUTE_UNUSED
 #endif
 
 // a quick macro for making sure that an object

src/shogun/io/fs/FileSystem.h

@@ -0,0 +1,193 @@
+/*
+ * This software is distributed under BSD 3-clause license (see LICENSE file).
+ *
+ * Authors: Viktor Gal
+ */
+
+#ifndef SHOGUN_FILESYSTEM_H
+#define SHOGUN_FILESYSTEM_H
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <shogun/base/macros.h>
+#include <shogun/lib/chunk.h>
+#include <shogun/io/fs/FileSystemRegistry.h>
+
+namespace shogun
+{
+class RandomAccessFile;
+class WritableFile;
+
+/**
+ * Interface representing a filesystem.
+ * A unified interface to open read and write files.
+ */
+class FileSystem
+{
+public:
+	FileSystem() {}
+
+	virtual ~FileSystem() {}
+
+	/**
+	 * Create a new random access read-only file
+	 *
+     * @param name file name string
+     * @return unique pointer to the file or exception in case of error
+     */
+	virtual std::unique_ptr<RandomAccessFile> new_random_access_file(const std::string& fname) = 0;
+
+	/**
+	 * Create a new writable file
+	 *
+     * @param name file name string
+     * @return unique pointer to the file or exception in case of error
+     */
+	virtual std::unique_ptr<WritableFile> new_writable_file(const std::string& fname) = 0;
+
+	/**
+	 * Create a new writable file starting on the end of the file
+	 *
+     * @param name file name string
+     * @return unique pointer to the file or exception in case of error
+     */
+	virtual std::unique_ptr<WritableFile> new_appendable_file(const std::string& fname) = 0;
+
+	/**
+	 * Check if file exists
+	 *
+	 * @return True in case the file exists False otherwise.
+	 */
+	virtual bool file_exists(const std::string& fname) = 0;
+
+	/**
+	 * Delete a given file
+	 *
+	 * @param fname file name to be deleted
+	 */
+	virtual void delete_file(const std::string& fname) = 0;
+
+	/**
+	 * Create directory
+	 *
+	 * @param dirname name of the directory to create
+	 */
+	virtual void create_dir(const std::string& dirname) = 0;
+
+	/**
+	 * Delete directory
+	 *
+	 * @param dirname name of the directory to delete
+	 */
+	virtual void delete_dir(const std::string& dirname) = 0;
+
+	/**
+	 * Rename file
+	 *
+	 * @param src file name to rename
+	 * @param target of the renaming
+	 */
+	virtual void rename_file(const std::string& src, const std::string& target) = 0;
+
+	/**
+	 * Translate name.
+	 * Resolves and cleans up the URI
+	 *
+	 * @param name the URI of the file
+	 */
+	virtual std::string translate_name(const std::string& name) const
+	{
+		// TODO: clean the path from junk
+		return name;
+	}
+
+	/**
+	 * Checks whether a given path is a directory or not.
+	 *
+	 * @param name path to the directory
+	 */
+	virtual bool is_directory(const std::string& fname) = 0;
+
+	virtual uint64_t get_file_size(const std::string& fname) = 0;
+};
+
+/**
+ * A file abstraction for randomly reading the contents of a file.
+ */
+class RandomAccessFile
+{
+public:
+	RandomAccessFile() {}
+	virtual ~RandomAccessFile() {}
+
+	virtual void read(
+		uint64_t offset, size_t n,
+		Chunk* result, char* scratch) const = 0;
+private:
+	SG_DELETE_COPY_AND_ASSIGN(RandomAccessFile);
+};
+
+/**
+ * A file abstraction for sequentially writing out a file.
+ */
+class WritableFile
+{
+public:
+	WritableFile() {}
+	virtual ~WritableFile() {}
+
+	/**
+	 * append data to file
+	 */
+	virtual void append(const void* data, size_t size) = 0;
+
+	/**
+	 * Close the file
+	 */
+	virtual void close() = 0;
+
+	/**
+	 * Flush the file
+	 */
+	virtual void flush() = 0;
+
+	/**
+	 * Sync the content of the file to the file system
+	 */
+	virtual void sync() = 0;
+
+private:
+	SG_DELETE_COPY_AND_ASSIGN(WritableFile);
+};
+
+namespace internal {
+
+template <typename Factory>
+struct FileSystemRegister
+{
+	FileSystemRegister(FileSystemRegistry* fsr, const std::string& scheme)
+	{
+		fsr->register_fs(scheme, []() -> FileSystem* { return new Factory; });
+	}
+};
+
+} // namespace internal
+
+#define REGISTER_FILE_SYSTEM_FACTORY(fsr, scheme, factory) \
+  REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, fsr, scheme, factory)
+#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, fsr, scheme, factory) \
+  REGISTER_FILE_SYSTEM_UNIQ(ctr, fsr, scheme, factory)
+#define REGISTER_FILE_SYSTEM_UNIQ(ctr, fsr, scheme, factory)   \
+  static ::shogun::internal::FileSystemRegister<factory>       \
+      register_ff##ctr SG_ATTRIBUTE_UNUSED =                   \
+          ::shogun::internal::FileSystemRegister<factory>(fsr, scheme);
+
+#define REGISTER_FILE_SYSTEM(scheme, factory) \
+	REGISTER_FILE_SYSTEM_FACTORY(FileSystemRegistry::instance(), scheme, factory);
+
+} // namespace shogun
+
+#endif

src/shogun/io/fs/FileSystemRegistry.cpp

@@ -0,0 +1,117 @@
+#include <shogun/io/fs/FileSystem.h>
+#include <shogun/io/fs/FileSystemRegistry.h>
+#include <shogun/lib/exception/ShogunException.h>
+
+using namespace shogun;
+using namespace std;
+
+void FileSystemRegistry::register_fs(
+	const string& scheme,
+	FileSystemRegistry::Factory factory)
+{
+	lock_guard<mutex> lock(m_mutex);
+	if (!m_fs_registry.emplace(
+		scheme,	unique_ptr<FileSystem>(factory())).second)
+	{
+		throw ShogunException("File factor for " + scheme + " already registered");
+	}
+}
+
+FileSystem* FileSystemRegistry::lookup(const string& scheme) const
+{
+	lock_guard<mutex> lock(m_mutex);
+	const auto found = m_fs_registry.find(scheme);
+	return (found == m_fs_registry.end())
+		? nullptr
+		: found->second.get();
+}
+
+FileSystem* FileSystemRegistry::get_file_system_for_file(
+	const string& fname) const
+{
+	string scheme, host, path;
+	//io::ParseURI(fname, &scheme, &host, &path);
+	FileSystem* fs = lookup(scheme);
+	if (!fs)
+	{
+		if (scheme.empty())
+		  scheme = "[local]";
+		throw ShogunException(
+			"File system scheme '" + scheme +
+			"' not implemented (file: '" + scheme + "')");
+	}
+	return fs;
+}
+
+
+vector<string> FileSystemRegistry::get_registered_file_system_schemes()
+{
+	vector<string> schemes;
+  	lock_guard<mutex> lock(m_mutex);
+  	for (const auto& e: m_fs_registry)
+    	schemes.push_back(e.first);
+    return schemes;
+}
+
+unique_ptr<RandomAccessFile> FileSystemRegistry::new_random_access_file(const string& fname)
+{
+	return get_file_system_for_file(fname)->new_random_access_file(fname);
+}
+
+
+unique_ptr<WritableFile> FileSystemRegistry::new_writable_file(const string& fname)
+{
+	return get_file_system_for_file(fname)->new_writable_file(fname);
+}
+
+unique_ptr<WritableFile> FileSystemRegistry::new_appendable_file(const string& fname)
+{
+	return get_file_system_for_file(fname)->new_appendable_file(fname);
+}
+
+bool FileSystemRegistry::file_exists(const string& fname)
+{
+	return get_file_system_for_file(fname)->file_exists(fname);
+}
+
+void FileSystemRegistry::delete_file(const string& fname)
+{
+	return get_file_system_for_file(fname)->delete_file(fname);
+}
+
+void FileSystemRegistry::create_dir(const string& dirname)
+{
+	return get_file_system_for_file(dirname)->create_dir(dirname);
+}
+
+void FileSystemRegistry::delete_dir(const string& dirname)
+{
+	return get_file_system_for_file(dirname)->delete_dir(dirname);
+}
+
+void FileSystemRegistry::rename_file(const string& src, const string& target)
+{
+	FileSystem* src_fs = get_file_system_for_file(src);
+	FileSystem* target_fs = get_file_system_for_file(src);
+	if (src_fs != target_fs)
+		throw ShogunException(
+			"Renaming "+ src +" to "+ target +" not implemented");
+
+	return src_fs->rename_file(src, target);
+}
+
+string FileSystemRegistry::translate_name(const string& name) const
+{
+	return get_file_system_for_file(name)->translate_name(name);
+}
+
+bool FileSystemRegistry::is_directory(const string& fname)
+{
+	return get_file_system_for_file(fname)->is_directory(fname);
+}
+
+uint64_t FileSystemRegistry::get_file_size(const string& fname)
+{
+	return get_file_system_for_file(fname)->get_file_size(fname);
+}
+