picolibc, POSIX_API: fwrite() expects __file but fopen() returns fd_entry

[Ruanxingzhi]

Describe the bug

When using picolibc with posix filesystem API, fwrite expects a __file object, but fopen returns an fd_entry, which does not have a put handler.

struct fd_entry {
	void *obj;
	const struct fd_op_vtable *vtable;
	atomic_t refcount;
	struct k_mutex lock;
	struct k_condvar cond;
	size_t offset;
	uint32_t mode;
};

struct __file {
	__ungetc_t unget;	/* ungetc() buffer */
	__uint8_t  flags;	/* flags, see below */
	int	(*put)(char, struct __file *);	/* function to write one char to device */
	int	(*get)(struct __file *);	/* function to read one char from device */
	int	(*flush)(struct __file *);	/* function to flush output to device */
#ifdef __STDIO_LOCKING
	_LOCK_RECURSIVE_T lock;
#endif
};

Regression

• This is a regression.

Steps to reproduce

Board: xiao_esp32c6/esp32c6/hpcore, for example.

prj.conf:

CONFIG_SERIAL=y
CONFIG_GPIO=y
CONFIG_LOG=y
CONFIG_CONSOLE=y

CONFIG_POSIX_API=y
CONFIG_ZVFS_OPEN_ADD_SIZE_POSIX=100
CONFIG_POSIX_FILE_SYSTEM=y

CONFIG_DISK_ACCESS=y
CONFIG_DISK_DRIVERS=y
CONFIG_DISK_DRIVER_FLASH=y
CONFIG_FILE_SYSTEM=y
CONFIG_FILE_SYSTEM_MKFS=y
CONFIG_FILE_SYSTEM_LITTLEFS=y
CONFIG_DEBUG_OPTIMIZATIONS=y

CONFIG_LOG_BUFFER_SIZE=16384

#include <zephyr/storage/disk_access.h>

#include "fcntl.h"
#include "zephyr/kernel.h"
#include "zephyr/console/console.h"
#include "lua.h"
#include "lauxlib.h"
#include "lualib.h"
#include "zephyr/fs/fs.h"
#include "zephyr/storage/flash_map.h"
#include "zephyr/fs/littlefs.h"
#include "stdlib.h"
#include "unistd.h"


// it works 
void test_disk_access() {
    const static struct flash_area *area;
    flash_area_open(FIXED_PARTITION_ID(storage_partition), &area);

    static struct flash_sector sec[1];
    uint32_t cnt = 1;
    flash_area_sectors(area, &cnt, sec);

    printf(
        "flash area :: ID = %d, offset = %u, size = %u, sector size = %d\n",
        FIXED_PARTITION_ID(storage_partition),
        FIXED_PARTITION_OFFSET(storage_partition),
        FIXED_PARTITION_SIZE(storage_partition),
        sec[0].fs_size
    );

#define DISK_NAME "mydisk"

    int ret = disk_access_init(DISK_NAME);
    printf("disk_access_init: ret = %d\n", ret);

    static uint8_t buf[4096];
    ret = disk_access_read(DISK_NAME, buf, 0, 1);
    printf("disk_access_read: ret = %d\n", ret);

    printf("first 4 byte: %02x %02x %02x %02x\n", buf[0], buf[1], buf[2], buf[3]);

    static const uint8_t s[4096] = "\xca\xfe\xba\xbe";
    ret = disk_access_write(DISK_NAME, s, 0, 1);
    printf("disk_access_write: ret = %d\n", ret);

    ret = disk_access_read(DISK_NAME, buf, 0, 1);
    printf("disk_access_read: ret = %d\n", ret);

    printf("first 4 byte: %02x %02x %02x %02x\n", buf[0], buf[1], buf[2], buf[3]);

    ret = disk_access_ioctl(DISK_NAME, DISK_IOCTL_CTRL_SYNC, NULL);
    printf("disk_access_ioctl SYNC: ret = %d\n", ret);

    fflush(stdout);
}



// it works
FS_LITTLEFS_DECLARE_DEFAULT_CONFIG(storage);
static struct fs_mount_t lfs_storage_mnt = {
    .type = FS_LITTLEFS,
    .fs_data = &storage,
    .storage_dev = (void *) FIXED_PARTITION_ID(storage_partition),
    .mnt_point = "/lfs",
};

void test_fs() {
    int ret = fs_mount(&lfs_storage_mnt);
    printf("fs_mount: ret = %d\n", ret);

    struct fs_file_t file;
    fs_file_t_init(&file);
    ret = fs_open(&file, "/lfs/hello.txt", FS_O_CREATE | FS_O_WRITE);
    printf("fs_open: ret = %d\n", ret);

    const char str[] = "hello, world!";
    ret = fs_write(&file, str, sizeof(str));
    printf("fs_write: ret = %d\n", ret);

    fs_close(&file);
    printf("fs_close: ret = %d\n", ret);

    fs_file_t_init(&file);
    ret = fs_open(&file, "/lfs/hello.txt", FS_O_READ);
    printf("fs_open: ret = %d\n", ret);

    static char buf[1024];
    ret = fs_read(&file, buf, sizeof(buf));
    printf("fs_read: ret = %d\n", ret);

    printf("%s\n", buf);
    fs_close(&file);

    fflush(stdout);
}

// it works
void test_fs_posix_read_write() {
    const int fd = open("/lfs/posix_raw.txt", O_CREAT | O_RDWR | O_TRUNC);
    printf("open: fd = %d, errno = %d (%s)\n", fd, errno, strerror(errno));

    const char *msg = "hello raw read/write";
    int ret = write(fd, msg, strlen(msg));
    printf("write: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));

    lseek(fd, 0, SEEK_SET);

    char buf[1024];
    ret = read(fd, buf, 100);
    printf("read: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));
    puts(buf);

    ret = close(fd);
    printf("close: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));
}

// bug here!
void test_fs_posix_fopen() {
    FILE *fp = fopen("/lfs/posix.txt", "w+");
    printf("fopen: fp = %p, errno = %d (%s)\n", fp, errno, strerror(errno));

    const char *msg = "hello world";
    int ret = fwrite(msg, sizeof(char), strlen(msg), fp);
    printf("fwrite: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));

    ret = fseek(fp, 0, SEEK_SET);
    printf("fseek: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));

    char buf[1024];
    ret = fread(buf, sizeof(char), 100, fp);
    printf("fread: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));

    ret = fclose(fp);
    printf("fclose: ret = %d, errno = %d (%s)\n", ret, errno, strerror(errno));
}

int main() {
    // test_disk_access();

    test_fs();
    test_fs_posix_read_write();
    test_fs_posix_fopen();
}

Relevant log output

fs_mount: ret = 0
fs_open: ret = 0
fs_write: ret = 14
fs_close: ret = 14
fs_open: ret = 0
fs_read: ret = 14
hello, world!
open: fd = 3, errno = 0 (Success)
write: ret = 20, errno = 0 (Success)
read: ret = 20, errno = 0 (Success)
hello raw read/write
close: ret = 0, errno = 0 (Success)
fopen: fp = 0x4080d480, errno = 0 (Success)
fwrite: ret = 0, errno = 0 (Success)
fseek: ret = -1, errno = 29 (Illegal seek)
fread: ret = 0, errno = 29 (Illegal seek)
fclose: ret = 0, errno = 29 (Illegal seek)



(gdb) x/10wx 0x4080d480
0x4080d480 <fdtable+144>:       0x40813130      0x4080d3c8      0x00000001      0x4080d48c
0x4080d490 <fdtable+160>:       0x4080d48c      0x00000000      0x00000000      0x00000000
0x4080d4a0 <fdtable+176>:       0x4080d4a0      0x4080d4a0

(gdb) x/100wx fdtable
0x4080d3f0 <fdtable>:           0x00000000      0x4280102c      0x00000001      0x4080d3fc
0x4080d400 <fdtable+16>:        0x4080d3fc      0x00000000      0x00000000      0x0000000e
0x4080d410 <fdtable+32>:        0x4080d410      0x4080d410      0x00000000      0x00000000
0x4080d420 <fdtable+48>:        0x00000000      0x4280102c      0x00000001      0x4080d42c
0x4080d430 <fdtable+64>:        0x4080d42c      0x00000000      0x00000000      0x0000000e
0x4080d440 <fdtable+80>:        0x4080d440      0x4080d440      0x00000000      0x00000000
0x4080d450 <fdtable+96>:        0x00000000      0x4280102c      0x00000001      0x4080d45c
0x4080d460 <fdtable+112>:       0x4080d45c      0x00000000      0x00000000      0x0000000e
0x4080d470 <fdtable+128>:       0x4080d470      0x4080d470      0x00000000      0x00000000
0x4080d480 <fdtable+144>:       0x40813130      0x4080d3c8      0x00000001      0x4080d48c
0x4080d490 <fdtable+160>:       0x4080d48c      0x00000000      0x00000000      0x00000000
0x4080d4a0 <fdtable+176>:       0x4080d4a0      0x4080d4a0      0x00000000      0x00000000
0x4080d4b0 <fdtable+192>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d4c0 <fdtable+208>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d4d0 <fdtable+224>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d4e0 <fdtable+240>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d4f0 <fdtable+256>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d500 <fdtable+272>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d510 <fdtable+288>:       0x00000000      0x00000000      0x00000000      0x00000000
0x4080d520 <fdtable+304>:       0x00000000      0x00000000      0x00000000      0x00000000


(gdb) p fdtable[0]
$4 = {obj = 0x0, vtable = 0x4280102c <stdinout_fd_op_vtable>, refcount = 1, lock = {wait_q = {waitq = {{head = 0x4080d3fc <fdtable+12>, next = 0x4080d3fc <fdtable+12>}, {tail = 0x4080d3fc <fdtable+12>, 
          prev = 0x4080d3fc <fdtable+12>}}}, owner = 0x0, lock_count = 0, owner_orig_prio = 14}, cond = {wait_q = {waitq = {{head = 0x4080d410 <fdtable+32>, next = 0x4080d410 <fdtable+32>}, {tail = 0x4080d410 <fdtable+32>, 
          prev = 0x4080d410 <fdtable+32>}}}}, offset = 0, mode = 0}
(gdb) p fdtable[1]
$5 = {obj = 0x0, vtable = 0x4280102c <stdinout_fd_op_vtable>, refcount = 1, lock = {wait_q = {waitq = {{head = 0x4080d42c <fdtable+60>, next = 0x4080d42c <fdtable+60>}, {tail = 0x4080d42c <fdtable+60>, 
          prev = 0x4080d42c <fdtable+60>}}}, owner = 0x0, lock_count = 0, owner_orig_prio = 14}, cond = {wait_q = {waitq = {{head = 0x4080d440 <fdtable+80>, next = 0x4080d440 <fdtable+80>}, {tail = 0x4080d440 <fdtable+80>, 
          prev = 0x4080d440 <fdtable+80>}}}}, offset = 0, mode = 0}
(gdb) p fdtable[2]
$6 = {obj = 0x0, vtable = 0x4280102c <stdinout_fd_op_vtable>, refcount = 1, lock = {wait_q = {waitq = {{head = 0x4080d45c <fdtable+108>, next = 0x4080d45c <fdtable+108>}, {tail = 0x4080d45c <fdtable+108>, 
          prev = 0x4080d45c <fdtable+108>}}}, owner = 0x0, lock_count = 0, owner_orig_prio = 14}, cond = {wait_q = {waitq = {{head = 0x4080d470 <fdtable+128>, next = 0x4080d470 <fdtable+128>}, {
          tail = 0x4080d470 <fdtable+128>, prev = 0x4080d470 <fdtable+128>}}}}, offset = 0, mode = 0}
(gdb) p fdtable[3]
$7 = {obj = 0x40813130 <_k_mem_slab_buf_file_desc_slab>, vtable = 0x4080d3c8 <posix_op_vtable>, refcount = 1, lock = {wait_q = {waitq = {{head = 0x4080d48c <fdtable+156>, next = 0x4080d48c <fdtable+156>}, {
          tail = 0x4080d48c <fdtable+156>, prev = 0x4080d48c <fdtable+156>}}}, owner = 0x0, lock_count = 0, owner_orig_prio = 0}, cond = {wait_q = {waitq = {{head = 0x4080d4a0 <fdtable+176>, next = 0x4080d4a0 <fdtable+176>}, {
          tail = 0x4080d4a0 <fdtable+176>, prev = 0x4080d4a0 <fdtable+176>}}}}, offset = 0, mode = 0}
(gdb) p fdtable[4]
$8 = {obj = 0x0, vtable = 0x0, refcount = 0, lock = {wait_q = {waitq = {{head = 0x0, next = 0x0}, {tail = 0x0, prev = 0x0}}}, owner = 0x0, lock_count = 0, owner_orig_prio = 0}, cond = {wait_q = {waitq = {{head = 0x0, 
          next = 0x0}, {tail = 0x0, prev = 0x0}}}}, offset = 0, mode = 0}

Impact

Functional Limitation – Some features not working as expected, but system usable.

Environment

Zephyr SDK 4.3.99

Additional Context

No response

[Ruanxingzhi]

A temporary workaround which overwrites fopen and fclose:

int parse_mode(const char *mode, fs_mode_t *optr) {
    int ret;
    fs_mode_t o;

    switch (mode[0]) {
        case 'r': /* open for reading */
            ret = __SRD;
            o = FS_O_READ;
            break;

        case 'w': /* open for writing */
            ret = __SWR;
            o = FS_O_WRITE | FS_O_CREATE | FS_O_TRUNC;
            break;

        case 'a': /* open for appending */
            ret = __SWR;
            o = FS_O_RDWR | FS_O_APPEND;
            break;

        default: /* illegal mode */
            errno = EINVAL;
            return (0);
    }
    while (*++mode) {
        switch (*mode) {
            case '+':
                ret |= (__SRD | __SWR);
                o |= FS_O_RDWR;
                break;
            default:
                break;
        }
    }
    *optr = o;
    return ret;
}

struct my_file_entry {
    struct __file io_file;
    struct fs_file_t fs_file;
};

#define GET_ENTRY(fp) ((struct my_file_entry *) fp)


int my_fs_put(char c, struct __file *f) {
    return fs_write(&GET_ENTRY(f)->fs_file, &c, 1);
}

int my_fs_get(struct __file *f) {
    char c;
    int ret = fs_read(&GET_ENTRY(f)->fs_file, &c, 1);

    // printk("call: fs_getc %p: c = %x, ret = %d\n", f, c, ret);

    if (ret == 0) {
         return _FDEV_EOF;
    }

    if (ret < 0) {
        return _FDEV_ERR;
    }

    return c;
}

int my_fs_flush(struct __file *f) {
    return fs_sync(&GET_ENTRY(f)->fs_file);
}

FILE *fopen(const char *filename, const char *mode) {
    struct my_file_entry *it = k_malloc(sizeof(struct my_file_entry));
    fs_file_t_init(&it->fs_file);

    fs_mode_t fs_flags;
    int io_flags = parse_mode(mode, &fs_flags);
    if (!io_flags) {
        return NULL;
    }

    if (fs_open(&it->fs_file, filename, fs_flags) < 0) {
        return NULL;
    }

    it->io_file.put = my_fs_put;
    it->io_file.get = my_fs_get;
    it->io_file.flush = my_fs_flush;
    it->io_file.flags = io_flags;
    return &it->io_file;
}


int fclose(FILE *fp) {
    struct my_file_entry *it = GET_ENTRY(fp);
    int ret = fs_close(&it->fs_file);
    k_free(it);

    if (ret < 0) {
        return -1;
    }
    return 0;
}

[keith-packard]

This whole subsystem is very confusing to me -- why is there any FILE object in Zephyr aside from the one provide by the C library?