diff --git a/extension/data_loader/file_data_loader.cpp b/extension/data_loader/file_data_loader.cpp index 9e0a958ea0d..b58edaa2dee 100644 --- a/extension/data_loader/file_data_loader.cpp +++ b/extension/data_loader/file_data_loader.cpp @@ -23,10 +23,20 @@ #include #include +// Some platforms (e.g. Xtensa) do not support pread() that we use to read the +// file at different offsets simultaneously from multiple threads not affecting +// each other. We list them below and use a workaround for them. +#if defined(__xtensa__) +#define ET_HAVE_PREAD 0 +#endif // defined(__xtensa__) + +#ifndef ET_HAVE_PREAD +#define ET_HAVE_PREAD 1 +#endif // !ET_HAVE_PREAD + namespace torch { namespace executor { namespace util { - namespace { /** @@ -226,29 +236,28 @@ ET_NODISCARD Error FileDataLoader::load_into( ET_CHECK_OR_RETURN_ERROR( buffer != nullptr, InvalidArgument, "Provided buffer cannot be null"); - // Seek to the right place in the file. - off_t seek_offset = ::lseek(fd_, offset, SEEK_SET); - if (seek_offset != offset) { - ET_LOG( - Error, - "Seeking %s to offset %zu returned %zd: %s", - file_name_, - offset, - (ssize_t)seek_offset, - strerror(errno)); - return Error::AccessFailed; - } - // Read the data into the aligned address. size_t needed = size; uint8_t* buf = reinterpret_cast(buffer); + + // Make a duplicate fd if pread() is not available and we have to seek(). + // Cannot use the standard dup() or fcntl() calls because the returned + // duplicate will share the underlying file record and affect the original fd + // when seeking on multiple threads simultaneously. + const auto dup_fd = ET_HAVE_PREAD ? fd_ : ::open(file_name_, O_RDONLY); + while (needed > 0) { - // Reads on macos will fail with EINVAL if size > INT32_MAX. - ssize_t nread = ::read( - fd_, - buf, - std::min( - needed, static_cast(std::numeric_limits::max()))); + // Reads on macOS will fail with EINVAL if size > INT32_MAX. + const auto chunk_size = std::min( + needed, static_cast(std::numeric_limits::max())); + const auto nread = +#if ET_HAVE_PREAD + ::pread(dup_fd, buf, chunk_size, offset); +#else + (::lseek(dup_fd, offset, SEEK_SET) == (off_t)-1) + ? -1 + : ::read(dup_fd, buf, chunk_size); +#endif if (nread < 0 && errno == EINTR) { // Interrupted by a signal; zero bytes read. continue; @@ -263,10 +272,17 @@ ET_NODISCARD Error FileDataLoader::load_into( size, offset, nread == 0 ? "EOF" : strerror(errno)); + if (!ET_HAVE_PREAD) { + ::close(dup_fd); + } return Error::AccessFailed; } needed -= nread; buf += nread; + offset += nread; + } + if (!ET_HAVE_PREAD) { + ::close(dup_fd); } return Error::Ok; }