WasiFileSystem.canonicalize (#1313)

* WasiFileSystem.canonicalize

Also appendingSink

Also deleting directories

I'm very eager to bring AbstractFileSystemTest online for this.

* Also recover if the errno is isdir

* Spotless

okio-wasifilesystem/src/wasmMain/kotlin/okio/WasiFileSystem.kt

@@ -20,10 +20,13 @@ import kotlin.wasm.unsafe.withScopedMemoryAllocator
 import okio.Path.Companion.toPath
 import okio.internal.ErrnoException
 import okio.internal.fdClose
+import okio.internal.preview1.Errno
 import okio.internal.preview1.FirstPreopenDirectoryTmp
 import okio.internal.preview1.dirnamelen
 import okio.internal.preview1.fd
 import okio.internal.preview1.fd_readdir
+import okio.internal.preview1.fdflags
+import okio.internal.preview1.fdflags_append
 import okio.internal.preview1.filetype
 import okio.internal.preview1.filetype_directory
 import okio.internal.preview1.filetype_regular_file
@@ -36,6 +39,7 @@ import okio.internal.preview1.path_create_directory
 import okio.internal.preview1.path_filestat_get
 import okio.internal.preview1.path_open
 import okio.internal.preview1.path_readlink
+import okio.internal.preview1.path_remove_directory
 import okio.internal.preview1.path_rename
 import okio.internal.preview1.path_symlink
 import okio.internal.preview1.path_unlink_file
@@ -53,7 +57,44 @@ import okio.internal.write
  */
 object WasiFileSystem : FileSystem() {
   override fun canonicalize(path: Path): Path {
-    TODO("Not yet implemented")
+    // There's no APIs in preview1 to canonicalize a path. We give it a best effort by resolving
+    // all symlinks, but this could result in a relative path.
+    val candidate = resolveSymlinks(path, 0)
+
+    if (!candidate.isAbsolute) {
+      throw IOException("WASI preview1 cannot canonicalize relative paths")
+    }
+
+    return candidate
+  }
+
+  private fun resolveSymlinks(
+    path: Path,
+    recurseCount: Int = 0,
+  ): Path {
+    // 40 is chosen for consistency with the Linux kernel (which previously used 8).
+    if (recurseCount > 40) throw IOException("symlink cycle?")
+
+    val parent = path.parent
+    val resolvedParent = when {
+      parent != null -> resolveSymlinks(parent, recurseCount + 1)
+      else -> null
+    }
+    val pathWithResolvedParent = when {
+      resolvedParent != null -> resolvedParent / path.name
+      else -> path
+    }
+
+    val symlinkTarget = metadata(pathWithResolvedParent).symlinkTarget
+      ?: return pathWithResolvedParent
+
+    val resolvedSymlinkTarget = when {
+      symlinkTarget.isAbsolute -> symlinkTarget
+      resolvedParent != null -> resolvedParent / symlinkTarget
+      else -> symlinkTarget
+    }
+
+    return resolveSymlinks(resolvedSymlinkTarget, recurseCount + 1)
   }
 
   override fun metadataOrNull(path: Path): FileMetadata? {
@@ -68,6 +109,13 @@ object WasiFileSystem : FileSystem() {
         pathSize = pathSize,
         returnPointer = returnPointer.address.toInt(),
       )
+
+      // When calling path_filestat_get on '/', don't crash.
+      when (errno) {
+        Errno.notcapable.ordinal -> return FileMetadata(isDirectory = true)
+        Errno.noent.ordinal -> throw FileNotFoundException("no such file: $path")
+      }
+
       if (errno != 0) throw ErrnoException(errno.toShort())
 
       // Skip device, offset 0.
@@ -213,7 +261,19 @@ object WasiFileSystem : FileSystem() {
   }
 
   override fun appendingSink(file: Path, mustExist: Boolean): Sink {
-    TODO("Not yet implemented")
+    val oflags = when {
+      mustExist -> 0
+      else -> oflag_creat
+    }
+
+    return FileSink(
+      fd = pathOpen(
+        path = file.toString(),
+        oflags = oflags,
+        rightsBase = right_fd_write,
+        fdflags = fdflags_append,
+      ),
+    )
   }
 
   override fun createDirectory(dir: Path, mustCreate: Boolean) {
@@ -252,11 +312,23 @@ object WasiFileSystem : FileSystem() {
     withScopedMemoryAllocator { allocator ->
       val (pathAddress, pathSize) = allocator.write(path.toString())
 
-      val errno = path_unlink_file(
+      var errno = path_unlink_file(
         fd = FirstPreopenDirectoryTmp,
         path = pathAddress.address.toInt(),
         pathSize = pathSize,
       )
+      // If unlink failed, try remove_directory.
+      when (errno) {
+        Errno.perm.ordinal,
+        Errno.isdir.ordinal,
+        -> {
+          errno = path_remove_directory(
+            fd = FirstPreopenDirectoryTmp,
+            path = pathAddress.address.toInt(),
+            pathSize = pathSize,
+          )
+        }
+      }
       if (errno != 0) throw ErrnoException(errno.toShort())
     }
   }
@@ -281,6 +353,7 @@ object WasiFileSystem : FileSystem() {
     path: String,
     oflags: oflags,
     rightsBase: rights,
+    fdflags: fdflags = 0,
   ): fd {
     withScopedMemoryAllocator { allocator ->
       val (pathAddress, pathSize) = allocator.write(path)
@@ -294,7 +367,7 @@ object WasiFileSystem : FileSystem() {
         oflags = oflags,
         fs_rights_base = rightsBase,
         fs_rights_inheriting = 0,
-        fdflags = 0,
+        fdflags = fdflags,
         returnPointer = returnPointer.address.toInt(),
       )
       if (errno != 0) throw ErrnoException(errno.toShort())

okio-wasifilesystem/src/wasmMain/kotlin/okio/internal/preview1/Fdflags.kt

@@ -0,0 +1,26 @@
+// Copyright 2019-2023 the Contributors to the WASI Specification
+// This file is adapted from the WASI preview1 spec here:
+// https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md
+package okio.internal.preview1
+
+/**
+ * `fdflags: Record`.
+ *
+ * File descriptor flags.
+ */
+typealias fdflags = Short
+
+/** Data written to the file is always appended to the file's end. */
+val fdflags_append: Short = (1 shl 0).toShort()
+
+/** Write according to synchronized I/O data integrity completion. Only the data stored in the file is synchronized. */
+val fdflags_dsync: Short = (1 shl 1).toShort()
+
+/** Non-blocking mode. */
+val fdflags_nonblock: Short = (1 shl 2).toShort()
+
+/** Synchronized read I/O operations. */
+val fdflags_rsync: Short = (1 shl 3).toShort()
+
+/** Write according to synchronized I/O file integrity completion. In addition to synchronizing the data stored in the file, the implementation may also synchronously update the file's metadata. */
+val fdflags_sync: Short = (1 shl 4).toShort()

okio-wasifilesystem/src/wasmMain/kotlin/okio/internal/preview1/Preview1.kt

@@ -52,19 +52,6 @@ typealias dirnamelen = Int
  */
 typealias PointerU8 = Int
 
-/**
- * `fdflags: Record`.
- *
- * File descriptor flags.
- *
- * bit0: append: Data written to the file is always appended to the file's end.
- * bit1: dsync: Write according to synchronized I/O data integrity completion. Only the data stored in the file is synchronized.
- * bit2: nonblock: Non-blocking mode.
- * bit3: rsync: bool Synchronized read I/O operations.
- * bit4: sync: bool Write according to synchronized I/O file integrity completion. In addition to synchronizing the data stored in the file, the implementation may also synchronously update the file's metadata.
- */
-typealias fdflags = Short
-
 val Stdin: fd = 0
 val Stdout: fd = 1
 val Stderr: fd = 2
@@ -144,6 +131,20 @@ internal external fun path_readlink(
   returnPointer: PointerU8,
 ): Int // should be Short??
 
+/**
+ * path_remove_directory(fd: fd, path: string) -> Result<(), errno>
+ *
+ * Remove a directory.
+ * Return [`errno::notempty`](#errno.notempty) if the directory is not empty.
+ * Note: This is similar to `unlinkat(fd, path, AT_REMOVEDIR)` in POSIX.
+ */
+@WasmImport("wasi_snapshot_preview1", "path_remove_directory")
+internal external fun path_remove_directory(
+  fd: fd,
+  path: PointerU8,
+  pathSize: size,
+): Int // should be Short??
+
 /**
  * path_rename(fd: fd, old_path: string, new_fd: fd, new_path: string) -> Result<(), errno>
  *

okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiTest.kt

@@ -18,6 +18,7 @@ package okio
 import kotlin.test.BeforeTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
 import kotlin.test.assertTrue
 import okio.ByteString.Companion.encodeUtf8
 import okio.Path.Companion.toPath
@@ -36,6 +37,58 @@ class WasiTest {
     fileSystem.createDirectory(base / "child")
   }
 
+  @Test
+  fun canonicalizeAbsolutePathNoSymlinks() {
+    val path = base / "regular_file.txt"
+    fileSystem.write(path) {
+      writeUtf8("hello")
+    }
+    assertEquals(
+      path,
+      fileSystem.canonicalize(path),
+    )
+  }
+
+  @Test
+  fun canonicalizeAbsolutePathWithSymlinksInFiles() {
+    val target = base / "target"
+    val source = base / "source"
+    fileSystem.write(target) {
+      writeUtf8("hello")
+    }
+    fileSystem.createSymlink(source, "target".toPath())
+    assertEquals(
+      target,
+      fileSystem.canonicalize(source),
+    )
+  }
+
+  @Test
+  fun canonicalizeAbsolutePathWithSymlinksInDirectories() {
+    val target = base / "target"
+    val source = base / "source"
+    fileSystem.createDirectory(target)
+    fileSystem.write(target / "file.txt") {
+      writeUtf8("hello")
+    }
+    fileSystem.createSymlink(source, "target".toPath())
+    assertEquals(
+      target / "file.txt",
+      fileSystem.canonicalize(source / "file.txt"),
+    )
+  }
+
+  @Test
+  fun canonicalizeAbsolutePathWithSymlinkCycle() {
+    fileSystem.createSymlink(base / "rock", "scissors".toPath())
+    fileSystem.createSymlink(base / "scissors", "paper".toPath())
+    fileSystem.createSymlink(base / "paper", "rock".toPath())
+    val e = assertFailsWith<IOException> {
+      fileSystem.canonicalize(base / "rock")
+    }
+    assertEquals("symlink cycle?", e.message)
+  }
+
   @Test
   fun writeAndReadEmptyFile() {
     writeAndReadFile(ByteString.EMPTY, base / "empty.txt")
@@ -82,6 +135,23 @@ class WasiTest {
     }
   }
 
+  @Test
+  fun appendToFile() {
+    val fileName = base / "append.txt"
+    fileSystem.write(fileName) {
+      writeUtf8("hello")
+    }
+    fileSystem.appendingSink(fileName).buffer().use {
+      it.writeUtf8(" world")
+    }
+    assertEquals(
+      "hello world",
+      fileSystem.read(fileName) {
+        readUtf8()
+      },
+    )
+  }
+
   @Test
   fun listDirectory() {
     fileSystem.write(base / "a") {
@@ -101,7 +171,7 @@ class WasiTest {
   }
 
   @Test
-  fun delete() {
+  fun deleteFile() {
     fileSystem.write(base / "a") {
     }
     fileSystem.write(base / "b") {
@@ -119,6 +189,22 @@ class WasiTest {
     )
   }
 
+  @Test
+  fun deleteDirectory() {
+    fileSystem.createDirectory(base / "a")
+    fileSystem.createDirectory(base / "b")
+    fileSystem.createDirectory(base / "c")
+    fileSystem.delete(base / "b")
+
+    assertEquals(
+      listOf(
+        base / "a",
+        base / "c",
+      ),
+      fileSystem.list(base).sorted(),
+    )
+  }
+
   @Test
   fun createSymlink() {
     val targetPath = base / "target"