Added VFS system for virtual filesystem backends

Tools/ardupilotwaf/boards.py

@@ -394,13 +394,22 @@ def configure_env(self, cfg, env):
             if not cfg.check_SFML(env):
                 cfg.fatal("Failed to find SFML libraries")
 
+        import fnmatch
         if cfg.options.sitl_osd:
-            env.CXXFLAGS += ['-DWITH_SITL_OSD','-DOSD_ENABLED=ENABLED','-DHAL_HAVE_AP_ROMFS_EMBEDDED_H']
-            import fnmatch
+            env.CXXFLAGS += ['-DWITH_SITL_OSD','-DOSD_ENABLED=1']
             for f in os.listdir('libraries/AP_OSD/fonts'):
                 if fnmatch.fnmatch(f, "font*bin"):
                     env.ROMFS_FILES += [(f,'libraries/AP_OSD/fonts/'+f)]
 
+        # embed any scripts from ROMFS/scripts
+        if os.path.exists('ROMFS/scripts'):
+            for f in os.listdir('ROMFS/scripts'):
+                if fnmatch.fnmatch(f, "*.lua"):
+                    env.ROMFS_FILES += [('scripts/'+f,'ROMFS/scripts/'+f)]
+
+        if len(env.ROMFS_FILES) > 0:
+            env.CXXFLAGS += ['-DHAL_HAVE_AP_ROMFS_EMBEDDED_H']
+
         if cfg.options.sitl_rgbled:
             env.CXXFLAGS += ['-DWITH_SITL_RGBLED']
 
@@ -571,12 +580,12 @@ def build(self, bld):
 
     def pre_build(self, bld):
         '''pre-build hook that gets called before dynamic sources'''
-        super(chibios, self).pre_build(bld)
         from waflib.Context import load_tool
         module = load_tool('chibios', [], with_sys_path=True)
         fun = getattr(module, 'pre_build', None)
         if fun:
             fun(bld)
+        super(chibios, self).pre_build(bld)
 
 class linux(Board):
     def configure_env(self, cfg, env):

Tools/ardupilotwaf/chibios.py

@@ -352,38 +352,50 @@ def bldpath(path):
         cfg.msg('Default parameters', cfg.options.default_parameters, color='YELLOW')
         env.DEFAULT_PARAMETERS = cfg.options.default_parameters
 
-    # we need to run chibios_hwdef.py at configure stage to generate the ldscript.ld
-    # that is needed by the remaining configure checks
+    try:
+        ret = generate_hwdef_h(env)
+    except Exception:
+        cfg.fatal("Failed to process hwdef.dat")
+    if ret != 0:
+        cfg.fatal("Failed to process hwdef.dat ret=%d" % ret)
+    load_env_vars(cfg.env)
+    if env.HAL_WITH_UAVCAN:
+        setup_can_build(cfg)
+    setup_optimization(cfg.env)
+
+def generate_hwdef_h(env):
+    '''run chibios_hwdef.py'''
     import subprocess
 
     if env.BOOTLOADER:
-        env.HWDEF = srcpath('libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef-bl.dat' % env.BOARD)
+        env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef-bl.dat' % env.BOARD)
         env.BOOTLOADER_OPTION="--bootloader"
     else:
-        env.HWDEF = srcpath('libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef.dat' % env.BOARD)
+        env.HWDEF = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/%s/hwdef.dat' % env.BOARD)
         env.BOOTLOADER_OPTION=""
-    hwdef_script = srcpath('libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py')
+    hwdef_script = os.path.join(env.SRCROOT, 'libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py')
     hwdef_out = env.BUILDROOT
     if not os.path.exists(hwdef_out):
         os.mkdir(hwdef_out)
     python = sys.executable
-    try:
-        cmd = "{0} '{1}' -D '{2}' '{3}' {4} --params '{5}'".format(python, hwdef_script, hwdef_out, env.HWDEF, env.BOOTLOADER_OPTION, cfg.options.default_parameters)
-        ret = subprocess.call(cmd, shell=True)
-    except Exception:
-        cfg.fatal("Failed to process hwdef.dat")
-    if ret != 0:
-        cfg.fatal("Failed to process hwdef.dat ret=%d" % ret)
-    load_env_vars(cfg.env)
-    if env.HAL_WITH_UAVCAN:
-        setup_can_build(cfg)
-    setup_optimization(cfg.env)
+    cmd = "{0} '{1}' -D '{2}' '{3}' {4} --params '{5}'".format(python, hwdef_script, hwdef_out, env.HWDEF, env.BOOTLOADER_OPTION, env.DEFAULT_PARAMETERS)
+    return subprocess.call(cmd, shell=True)
 
 def pre_build(bld):
     '''pre-build hook to change dynamic sources'''
     load_env_vars(bld.env)
     if bld.env.HAL_WITH_UAVCAN:
         bld.get_board().with_uavcan = True
+    hwdef_h = os.path.join(bld.env.BUILDROOT, 'hwdef.h')
+    if not os.path.exists(hwdef_h):
+        print("Generating hwdef.h")
+        try:
+            ret = generate_hwdef_h(bld.env)
+        except Exception:
+            bld.fatal("Failed to process hwdef.dat")
+        if ret != 0:
+            bld.fatal("Failed to process hwdef.dat ret=%d" % ret)
+    setup_optimization(bld.env)
 
 def build(bld):
 
@@ -394,7 +406,8 @@ def build(bld):
             bld.env.get_flat('PYTHON'), bld.env.HWDEF, bld.env.BOOTLOADER_OPTION, bld.env.default_parameters),
         group='dynamic_sources',
         target=[bld.bldnode.find_or_declare('hwdef.h'),
-                bld.bldnode.find_or_declare('ldscript.ld')]
+                bld.bldnode.find_or_declare('ldscript.ld'),
+                bld.bldnode.find_or_declare('hw.dat')]
     )
 
     bld(
@@ -405,6 +418,7 @@ def build(bld):
     )
 
     common_src = [bld.bldnode.find_or_declare('hwdef.h'),
+                  bld.bldnode.find_or_declare('hw.dat'),
                   bld.bldnode.find_or_declare('modules/ChibiOS/include_dirs')]
     common_src += bld.path.ant_glob('libraries/AP_HAL_ChibiOS/hwdef/common/*.[ch]')
     common_src += bld.path.ant_glob('libraries/AP_HAL_ChibiOS/hwdef/common/*.mk')

Tools/ardupilotwaf/embed.py

@@ -68,6 +68,9 @@ def create_embedded_h(filename, files, uncompressed=False):
     out = open(filename, "wb")
     write_encode(out, '''// generated embedded files for AP_ROMFS\n\n''')
 
+    # remove duplicates and sort
+    files = sorted(list(set(files)))
+
     for i in range(len(files)):
         (name, filename) = files[i]
         if not embed_file(out, filename, i, name, uncompressed):

libraries/AP_Filesystem/AP_Filesystem.cpp

@@ -19,6 +19,190 @@
 
 static AP_Filesystem fs;
 
+#if CONFIG_HAL_BOARD == HAL_BOARD_CHIBIOS
+#include "AP_Filesystem_FATFS.h"
+static AP_Filesystem_FATFS fs_local;
+#endif
+#if CONFIG_HAL_BOARD == HAL_BOARD_LINUX || CONFIG_HAL_BOARD == HAL_BOARD_SITL
+#include "AP_Filesystem_posix.h"
+static AP_Filesystem_Posix fs_local;
+#endif
+
+#ifdef HAL_HAVE_AP_ROMFS_EMBEDDED_H
+#include "AP_Filesystem_ROMFS.h"
+static AP_Filesystem_ROMFS fs_romfs;
+#endif
+
+/*
+  mapping from filesystem prefix to backend
+ */
+const AP_Filesystem::Backend AP_Filesystem::backends[] = {
+    { nullptr, fs_local },
+#ifdef HAL_HAVE_AP_ROMFS_EMBEDDED_H
+    { "@ROMFS/", fs_romfs },
+#endif
+};
+
+#define MAX_FD_PER_BACKEND 256U
+#define NUM_BACKENDS ARRAY_SIZE(backends)
+#define LOCAL_BACKEND backends[0];
+#define BACKEND_IDX(backend) (&(backend) - &backends[0])
+
+/*
+  find backend by path
+ */
+const AP_Filesystem::Backend &AP_Filesystem::backend_by_path(const char *&path) const
+{
+    for (uint8_t i=1; i<NUM_BACKENDS; i++) {
+        const uint8_t plen = strlen(backends[i].prefix);
+        if (strncmp(path, backends[i].prefix, plen) == 0) {
+            path += plen;
+            return backends[i];
+        }
+    }
+    // default to local filesystem
+    return LOCAL_BACKEND;
+}
+
+/*
+  return backend by file descriptor
+ */
+const AP_Filesystem::Backend &AP_Filesystem::backend_by_fd(int &fd) const
+{
+    if (fd < 0 || uint32_t(fd) >= NUM_BACKENDS*MAX_FD_PER_BACKEND) {
+        return LOCAL_BACKEND;
+    }
+    const uint8_t idx = uint32_t(fd) / MAX_FD_PER_BACKEND;
+    fd -= idx * MAX_FD_PER_BACKEND;
+    return backends[idx];
+}
+
+int AP_Filesystem::open(const char *fname, int flags)
+{
+    const Backend &backend = backend_by_path(fname);
+    int fd = backend.fs.open(fname, flags);
+    if (fd < 0) {
+        return -1;
+    }
+    if (uint32_t(fd) >= MAX_FD_PER_BACKEND) {
+        backend.fs.close(fd);
+        errno = ERANGE;
+        return -1;
+    }
+    // offset fd so we can recognise the backend
+    const uint8_t idx = (&backend - &backends[0]);
+    fd += idx * MAX_FD_PER_BACKEND;
+    return fd;
+}
+
+int AP_Filesystem::close(int fd)
+{
+    const Backend &backend = backend_by_fd(fd);
+    return backend.fs.close(fd);
+}
+
+ssize_t AP_Filesystem::read(int fd, void *buf, size_t count)
+{
+    const Backend &backend = backend_by_fd(fd);
+    return backend.fs.read(fd, buf, count);
+}
+
+ssize_t AP_Filesystem::write(int fd, const void *buf, size_t count)
+{
+    const Backend &backend = backend_by_fd(fd);
+    return backend.fs.write(fd, buf, count);
+}
+
+int AP_Filesystem::fsync(int fd)
+{
+    const Backend &backend = backend_by_fd(fd);
+    return backend.fs.fsync(fd);
+}
+
+off_t AP_Filesystem::lseek(int fd, off_t offset, int seek_from)
+{
+    const Backend &backend = backend_by_fd(fd);
+    return backend.fs.lseek(fd, offset, seek_from);
+}
+
+int AP_Filesystem::stat(const char *pathname, struct stat *stbuf)
+{
+    const Backend &backend = backend_by_path(pathname);
+    return backend.fs.stat(pathname, stbuf);
+}
+
+int AP_Filesystem::unlink(const char *pathname)
+{
+    const Backend &backend = backend_by_path(pathname);
+    return backend.fs.unlink(pathname);
+}
+
+int AP_Filesystem::mkdir(const char *pathname)
+{
+    const Backend &backend = backend_by_path(pathname);
+    return backend.fs.mkdir(pathname);
+}
+
+AP_Filesystem::DirHandle *AP_Filesystem::opendir(const char *pathname)
+{
+    const Backend &backend = backend_by_path(pathname);
+    DirHandle *h = new DirHandle;
+    if (!h) {
+        return nullptr;
+    }
+    h->dir = backend.fs.opendir(pathname);
+    if (h->dir == nullptr) {
+        delete h;
+        return nullptr;
+    }
+    h->fs_index = BACKEND_IDX(backend);
+    return h;
+}
+
+struct dirent *AP_Filesystem::readdir(DirHandle *dirp)
+{
+    if (!dirp) {
+        return nullptr;
+    }
+    const Backend &backend = backends[dirp->fs_index];
+    return backend.fs.readdir(dirp->dir);
+}
+
+int AP_Filesystem::closedir(DirHandle *dirp)
+{
+    if (!dirp) {
+        return -1;
+    }
+    const Backend &backend = backends[dirp->fs_index];
+    int ret = backend.fs.closedir(dirp->dir);
+    delete dirp;
+    return ret;
+}
+
+// return free disk space in bytes
+int64_t AP_Filesystem::disk_free(const char *path)
+{
+    const Backend &backend = backend_by_path(path);
+    return backend.fs.disk_free(path);
+}
+
+// return total disk space in bytes
+int64_t AP_Filesystem::disk_space(const char *path)
+{
+    const Backend &backend = backend_by_path(path);
+    return backend.fs.disk_space(path);
+}
+
+
+/*
+  set mtime on a file
+ */
+bool AP_Filesystem::set_mtime(const char *filename, const time_t mtime_sec)
+{
+    const Backend &backend = backend_by_path(filename);
+    return backend.fs.set_mtime(filename, mtime_sec);
+}
+
 namespace AP
 {
 AP_Filesystem &FS()

libraries/AP_Filesystem/AP_Filesystem.h

@@ -33,6 +33,11 @@
 #endif
 
 class AP_Filesystem {
+private:
+    struct DirHandle {
+        uint8_t fs_index;
+        void *dir;
+    };
 
 public:
     AP_Filesystem() {}
@@ -47,9 +52,10 @@ class AP_Filesystem {
     int stat(const char *pathname, struct stat *stbuf);
     int unlink(const char *pathname);
     int mkdir(const char *pathname);
-    DIR *opendir(const char *pathname);
-    struct dirent *readdir(DIR *dirp);
-    int closedir(DIR *dirp);
+
+    DirHandle *opendir(const char *pathname);
+    struct dirent *readdir(DirHandle *dirp);
+    int closedir(DirHandle *dirp);
 
     // return free disk space in bytes, -1 on error
     int64_t disk_free(const char *path);
@@ -59,6 +65,23 @@ class AP_Filesystem {
 
     // set modification time on a file
     bool set_mtime(const char *filename, const time_t mtime_sec);
+
+private:
+    struct Backend {
+        const char *prefix;
+        AP_Filesystem_Backend &fs;
+    };
+    static const struct Backend backends[];
+
+    /*
+      find backend by path
+     */
+    const Backend &backend_by_path(const char *&path) const;
+
+    /*
+      find backend by open fd
+     */
+    const Backend &backend_by_fd(int &fd) const;
 };
 
 namespace AP {