Merge 59f9846 into be97c7b

.github/workflows/lint.yml

@@ -59,7 +59,7 @@ jobs:
         run: |
           rustup component add rustfmt
           cargo install --force --version 0.15.0 cbindgen
-          cargo install --force --version 0.55.1 bindgen
+          cargo install --force --version 0.56.0 bindgen
       - name: Check bindings
         run: |
           mkdir build

CMakeLists.txt

@@ -116,6 +116,7 @@ option(SHADOW_TEST "build tests (default: OFF)" OFF)
 option(SHADOW_EXPORT "export service libraries and headers (default: OFF)" OFF)
 option(SHADOW_WERROR "turn compiler warnings into errors. (default: OFF)" OFF)
 option(SHADOW_COVERAGE "enable code-coverage instrumentation. (default: OFF)" OFF)
+option(SHADOW_USE_C_SYSCALLS "use only the C syscall handlers. (default: OFF)" OFF)
 
 ## display selected user options
 MESSAGE(STATUS)
@@ -165,6 +166,10 @@ else()
     set(CARGO_ENV_VARS "")
 endif(SHADOW_COVERAGE STREQUAL ON)
 
+if(SHADOW_USE_C_SYSCALLS STREQUAL ON)
+    add_definitions(-DUSE_C_SYSCALLS)
+endif()
+
 if($ENV{VERBOSE})
     add_definitions(-DVERBOSE)
 endif()

docs/5-Developer-Guide.md

@@ -6,6 +6,10 @@ When required, you can rebuild all of the C-Rust bindings by running:
 cd build && cmake --target bindings .. && make bindings
 ```
 
+To see the specific options/flags provided to bindgen and cbindgen, you can use `make VERBOSE=1 bindings`.
+
+Since the C bindings and Rust bindings rely on each other, you may sometimes need to build the bindings in a specific order. Instead of `make bindings`, you can be more specific using for example `make bindings_main_rust` to make the Rust bindings for `src/main`.
+
 You may need to install bindgen, cbindgen, and clang:
 
 ```bash

setup

@@ -67,6 +67,11 @@ def main():
         action="store_true", dest="do_coverage",
         default=False)
 
+    parser_build.add_argument('--use-c-syscalls',
+        help="use only the C syscall handlers (disables Rust syscall handlers)",
+        action="store_true", dest="do_use_c_syscalls",
+        default=False)
+
     parser_build.add_argument('-v', '--verbose',
         help="print verbose output from the compiler",
         action="store_true", dest="do_verbose",
@@ -194,6 +199,7 @@ def build(args, remaining):
     if args.disable_tgen: cmake_cmd += " -DBUILD_TGEN=OFF"
     if args.do_valgrind: cmake_cmd += " -DLOADER_VALGRIND=ON"
     if args.do_werror: cmake_cmd += " -DSHADOW_WERROR=ON"
+    if args.do_use_c_syscalls: cmake_cmd += " -DSHADOW_USE_C_SYSCALLS=ON"
 
     if args.do_coverage:
         if not args.do_debug:

src/main/Cargo.toml

@@ -9,6 +9,8 @@ path = "lib.rs"
 crate-type = ["staticlib"]
 
 [dependencies]
+atomic_refcell = "0.1"
+bitflags = "1.2"
 lazy_static = "1.4.0"
 libc = "0.2"
 log = "0.4"

src/main/bindings/CMakeLists.txt

@@ -3,8 +3,15 @@ add_subdirectory(rust)
 
 # A fake target that depends on the wrapper.
 add_custom_target(bindings_main)
-add_dependencies(bindings_main bindings_main_rust)
 add_dependencies(bindings_main bindings_main_c)
+add_dependencies(bindings_main bindings_main_rust)
+
+# The C bindings should be generated first since cbindgen doesn't require the Rust code
+# to be valid, whereas bindgen does require the C code to be valid. If the C bindings
+# are no longer correct, but the Rust bindings are generated first, then there will be
+# no way to correct the C bindings since the Rust binding generation will always fail
+# before the C bindings can be corrected.
+add_dependencies(bindings_main_rust bindings_main_c)
 
 # Only re-generate bindings when explicititly requested, so that
 # our CI doesn't need to install the heavy bindgen dependency.

src/main/bindings/c/CMakeLists.txt

@@ -17,14 +17,20 @@ foreach(COMPILE_DEFINITION IN LISTS COMPILE_DEFINITIONS)
     list(APPEND LLVM_FLAGS "-D${COMPILE_DEFINITION}")
 endforeach(COMPILE_DEFINITION)
 
-# Generate wrapper.rs in the source tree.
+# Generate bindings.h in the source tree.
 add_custom_command(OUTPUT bindings.h
     COMMAND cbindgen ${CMAKE_SOURCE_DIR}/src/main
         --config ${CMAKE_CURRENT_SOURCE_DIR}/cbindgen.toml
         --output ${CMAKE_CURRENT_SOURCE_DIR}/bindings.h)
 
+# Generate bindings-opaque.h in the source tree.
+add_custom_command(OUTPUT bindings-opaque.h
+    COMMAND cbindgen ${CMAKE_SOURCE_DIR}/src/main
+        --config ${CMAKE_CURRENT_SOURCE_DIR}/cbindgen-opaque.toml
+        --output ${CMAKE_CURRENT_SOURCE_DIR}/bindings-opaque.h)
+
 # A fake target that depends on the wrapper.
-add_custom_target(bindings_main_c DEPENDS bindings.h)
+add_custom_target(bindings_main_c DEPENDS bindings.h bindings-opaque.h)
 
 # Only re-generate bindings when explicititly requested, so that
 # our CI doesn't need to install the heavy bindgen dependency.

src/main/bindings/c/bindings-opaque.h

@@ -0,0 +1,38 @@
+/*
+ * The Shadow Simulator
+ * See LICENSE for licensing information
+ */
+// clang-format off
+
+
+#ifndef main_opaque_bindings_h
+#define main_opaque_bindings_h
+
+/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */
+
+// A queue of byte chunks.
+typedef struct ByteQueue ByteQueue;
+
+typedef struct CompatDescriptor CompatDescriptor;
+
+// Manages the address-space for a plugin process.
+//
+// The MemoryManager's primary purpose is to make plugin process's memory directly accessible to
+// Shadow. It does this by tracking what regions of program memory in the plugin are mapped to
+// what (analagous to /proc/<pid>/maps), and *remapping* parts of the plugin's address space into
+// a shared memory-file, which is also mapped into Shadow.
+//
+// Shadow provides several methods for allowing Shadow to access the plugin's memory, such as
+// `get_readable_ptr`. If the corresponding region of plugin memory is mapped into the shared
+// memory file, the corresponding Shadow pointer is returned. If not, then, it'll fall back to
+// (generally slower) Thread APIs.
+//
+// For the MemoryManager to maintain consistent state, and to remap regions of memory it knows how
+// to remap, Shadow must delegate handling of mman-related syscalls (such as `mmap`) to the
+// MemoryManager via its `handle_*` methods.
+typedef struct MemoryManager MemoryManager;
+
+// An opaque type used when passing `*const AtomicRefCell<File>` to C.
+typedef struct PosixFileArc PosixFileArc;
+
+#endif /* main_opaque_bindings_h */

src/main/bindings/c/bindings.h

@@ -14,31 +14,58 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdlib.h>
+#include "main/bindings/c/bindings-opaque.h"
+#include "main/host/descriptor/descriptor_types.h"
+#include "main/host/status_listener.h"
+#include "main/host/syscall_handler.h"
 #include "main/host/syscall_types.h"
 #include "main/host/thread.h"
 
-// A queue of byte chunks.
-typedef struct ByteQueue ByteQueue;
-
-// Manages the address-space for a plugin process.
-//
-// The MemoryManager's primary purpose is to make plugin process's memory directly accessible to
-// Shadow. It does this by tracking what regions of program memory in the plugin are mapped to
-// what (analagous to /proc/<pid>/maps), and *remapping* parts of the plugin's address space into
-// a shared memory-file, which is also mapped into Shadow.
-//
-// Shadow provides several methods for allowing Shadow to access the plugin's memory, such as
-// `get_readable_ptr`. If the corresponding region of plugin memory is mapped into the shared
-// memory file, the corresponding Shadow pointer is returned. If not, then, it'll fall back to
-// (generally slower) Thread APIs.
-//
-// For the MemoryManager to maintain consistent state, and to remap regions of memory it knows how
-// to remap, Shadow must delegate handling of mman-related syscalls (such as `mmap`) to the
-// MemoryManager via its `handle_*` methods.
-typedef struct MemoryManager MemoryManager;
-
 void rust_logging_init(void);
 
+// The new compat descriptor takes ownership of the reference to the legacy descriptor and
+// does not increment its ref count, but will decrement the ref count when this compat
+// descriptor is freed/dropped.
+CompatDescriptor *compatdescriptor_fromLegacy(LegacyDescriptor *legacy_descriptor);
+
+// If the compat descriptor is a legacy descriptor, returns a pointer to the legacy
+// descriptor object. Otherwise returns NULL. The legacy descriptor's ref count is not
+// modified, so the pointer must not outlive the lifetime of the compat descriptor.
+LegacyDescriptor *compatdescriptor_asLegacy(const CompatDescriptor *descriptor);
+
+// When the compat descriptor is freed/dropped, it will decrement the legacy descriptor's
+// ref count.
+void compatdescriptor_free(CompatDescriptor *descriptor);
+
+// This is a no-op for non-legacy descriptors.
+void compatdescriptor_setHandle(CompatDescriptor *descriptor, int handle);
+
+// If the compat descriptor is a new descriptor, returns a pointer to the reference-counted
+// posix file object. Otherwise returns NULL. The posix file object's ref count is not
+// modified, so the pointer must not outlive the lifetime of the compat descriptor.
+const PosixFileArc *compatdescriptor_borrowPosixFile(CompatDescriptor *descriptor);
+
+// If the compat descriptor is a new descriptor, returns a pointer to the reference-counted
+// posix file object. Otherwise returns NULL. The posix file object's ref count is
+// incremented, so the pointer must always later be passed to `posixfile_drop()`, otherwise
+// the memory will leak.
+const PosixFileArc *compatdescriptor_newRefPosixFile(CompatDescriptor *descriptor);
+
+// Decrement the ref count of the posix file object. The pointer must not be used after
+// calling this function.
+void posixfile_drop(const PosixFileArc *file);
+
+// Get the status of the posix file object.
+Status posixfile_getStatus(const PosixFileArc *file);
+
+// Add a status listener to the posix file object. This will increment the status
+// listener's ref count, and will decrement the ref count when this status listener is
+// removed or when the posix file is freed/dropped.
+void posixfile_addListener(const PosixFileArc *file, StatusListener *listener);
+
+// Remove a listener from the posix file object.
+void posixfile_removeListener(const PosixFileArc *file, StatusListener *listener);
+
 // # Safety
 // * `thread` must point to a valid object.
 MemoryManager *memorymanager_new(Thread *thread);
@@ -106,10 +133,22 @@ SysCallReg memorymanager_handleMprotect(MemoryManager *memory_manager,
                                         uintptr_t size,
                                         int32_t prot);
 
+SysCallReturn rustsyscallhandler_close(SysCallHandler *sys, const SysCallArgs *args);
+
+SysCallReturn rustsyscallhandler_read(SysCallHandler *sys, const SysCallArgs *args);
+
+SysCallReturn rustsyscallhandler_write(SysCallHandler *sys, const SysCallArgs *args);
+
+SysCallReturn rustsyscallhandler_pipe(SysCallHandler *sys, const SysCallArgs *args);
+
+SysCallReturn rustsyscallhandler_pipe2(SysCallHandler *sys, const SysCallArgs *args);
+
 ByteQueue *bytequeue_new(size_t chunk_size);
 
 void bytequeue_free(ByteQueue *bq_ptr);
 
+size_t bytequeue_len(ByteQueue *bq);
+
 void bytequeue_push(ByteQueue *bq, const unsigned char *src, size_t len);
 
 size_t bytequeue_pop(ByteQueue *bq, unsigned char *dst, size_t len);

src/main/bindings/c/cbindgen-opaque.toml

@@ -0,0 +1,18 @@
+language = 'C'
+include_guard = "main_opaque_bindings_h"
+line_length = 100
+documentation_style = "c99"
+header = '''/*
+ * The Shadow Simulator
+ * See LICENSE for licensing information
+ */
+// clang-format off
+'''
+autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */"
+no_includes = true
+
+[export]
+# Avoid exporting C types back through again.
+exclude = ["PluginPtr", "SysCallReg"]
+# Generate only opaque types
+item_types = ["opaque"]

src/main/bindings/c/cbindgen.toml

@@ -9,8 +9,17 @@ header = '''/*
 // clang-format off
 '''
 autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */"
-includes = ["main/host/syscall_types.h", "main/host/thread.h"]
+includes = [
+  "main/bindings/c/bindings-opaque.h",
+  "main/host/descriptor/descriptor_types.h",
+  "main/host/status_listener.h",
+  "main/host/syscall_handler.h",
+  "main/host/syscall_types.h",
+  "main/host/thread.h",
+]
 
 [export]
 # Avoid exporting C types back through again.
 exclude = ["PluginPtr", "SysCallReg"]
+# Generate all item types, excluding opaque types
+item_types = ["constants", "globals", "enums", "structs", "unions", "typedefs", "functions"]

src/main/bindings/rust/CMakeLists.txt

@@ -21,10 +21,33 @@ endforeach(COMPILE_DEFINITION)
 add_custom_command(OUTPUT wrapper.rs
     COMMAND bindgen
         --whitelist-function "thread_.*"
+        --whitelist-function "descriptor_unref"
+        --whitelist-function "descriptor_setHandle"
+        --whitelist-function "process_.*CompatDescriptor"
+        --whitelist-function "process_get.*Ptr"
+        --whitelist-function "statuslistener_ref"
+        --whitelist-function "statuslistener_unref"
+        --whitelist-function "statuslistener_onStatusChanged"
+        --whitelist-function "syscallcondition_new"
+        --whitelist-function "syscallhandler_.*"
+        --blacklist-function "syscallhandler_new"
+        --blacklist-function "syscallhandler_ref"
+        --blacklist-function "syscallhandler_unref"
+        --blacklist-function "syscallhandler_make_syscall"
         --whitelist-type "PluginPtr"
+        --whitelist-type "Status"
+        --whitelist-type "StatusListener"
         --whitelist-type "SysCall.*"
+        --whitelist-type "LegacyDescriptor"
+        --whitelist-type "Trigger"
+        --whitelist-type "TriggerType"
+        --whitelist-var "CONFIG_PIPE_BUFFER_SIZE"
+        --whitelist-var "SYSCALL_IO_BUFSIZE"
+        --opaque-type "LegacyDescriptor"
+        --opaque-type "CompatDescriptor"
         --disable-header-comment
         --raw-line "/* automatically generated by rust-bindgen */"
+        --raw-line "use crate::host::descriptor::CompatDescriptor\;"
         ${CMAKE_CURRENT_SOURCE_DIR}/wrapper.h -o ${CMAKE_CURRENT_SOURCE_DIR}/wrapper.rs -- ${LLVM_FLAGS}
     MAIN_DEPENDENCY wrapper.h
     IMPLICIT_DEPENDS C wrapper.h)

src/main/bindings/rust/wrapper.h

@@ -5,5 +5,10 @@
 
 // Don't forget to whitelist functions/types/vars in CMakeLists.txt
 
+#include "main/host/descriptor/descriptor.h"
+#include "main/host/status.h"
+#include "main/host/status_listener.h"
+#include "main/host/syscall_condition.h"
 #include "main/host/syscall_types.h"
+#include "main/host/syscall/unistd.h"
 #include "main/host/thread.h"