Merge pull request RustPython#4782 from youknowone/bytes-as-osstr

relocate FsPath from stdlib::os and rename PyPathLike to OsPath

Author: fanninpm

common/src/os.rs

@@ -1,6 +1,6 @@
 // TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here
 
-use std::io;
+use std::{io, str::Utf8Error};
 
 #[cfg(windows)]
 pub fn errno() -> io::Error {
@@ -21,3 +21,19 @@ pub fn errno() -> io::Error {
 pub fn errno() -> io::Error {
     io::Error::last_os_error()
 }
+
+#[cfg(unix)]
+pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> {
+    use std::os::unix::ffi::OsStrExt;
+    Ok(std::ffi::OsStr::from_bytes(b))
+}
+
+#[cfg(not(unix))]
+pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> {
+    Ok(std::str::from_utf8(b)?.as_ref())
+}
+
+#[cfg(unix)]
+pub use std::os::unix::ffi;
+#[cfg(target_os = "wasi")]
+pub use std::os::wasi::ffi;

stdlib/src/posixsubprocess.rs

@@ -1,7 +1,7 @@
 use crate::vm::{
     builtins::PyListRef,
     function::ArgSequence,
-    stdlib::{os::PyPathLike, posix},
+    stdlib::{os::OsPath, posix},
     {PyObjectRef, PyResult, TryFromObject, VirtualMachine},
 };
 use nix::{errno::Errno, unistd};
@@ -60,7 +60,7 @@ struct CStrPathLike {
 }
 impl TryFromObject for CStrPathLike {
     fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
-        let s = PyPathLike::try_from_object(vm, obj)?.into_cstring(vm)?;
+        let s = OsPath::try_from_object(vm, obj)?.into_cstring(vm)?;
         Ok(CStrPathLike { s })
     }
 }

stdlib/src/socket.rs

@@ -14,7 +14,7 @@ mod _socket {
     use crate::vm::{
         builtins::{PyBaseExceptionRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef},
         convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject},
-        function::{ArgBytesLike, ArgMemoryBuffer, Either, OptionalArg, OptionalOption},
+        function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption},
         types::{DefaultConstructor, Initializer},
         utils::ToCString,
         AsObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
@@ -1977,12 +1977,10 @@ mod _socket {
 
     #[cfg(not(target_os = "redox"))]
     #[pyfunction]
-    fn if_nametoindex(name: PyObjectRef, vm: &VirtualMachine) -> PyResult<IfIndex> {
-        let name = crate::vm::stdlib::os::FsPath::try_from(name, true, vm)?;
-        let name = ffi::CString::new(name.as_bytes()).map_err(|err| err.into_pyexception(vm))?;
+    fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult<IfIndex> {
+        let name = name.to_cstring(vm)?;
 
         let ret = unsafe { c::if_nametoindex(name.as_ptr()) };
-
         if ret == 0 {
             Err(vm.new_os_error("no interface with this name".to_owned()))
         } else {

stdlib/src/sqlite.rs

@@ -58,10 +58,9 @@ mod _sqlite {
             PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef,
         },
         convert::IntoObject,
-        function::{ArgCallable, ArgIterable, FuncArgs, OptionalArg, PyComparisonValue},
+        function::{ArgCallable, ArgIterable, FsPath, FuncArgs, OptionalArg, PyComparisonValue},
         protocol::{PyBuffer, PyIterReturn, PyMappingMethods, PySequence, PySequenceMethods},
         sliceable::{SaturatedSliceIter, SliceableSequenceOp},
-        stdlib::os::PyPathLike,
         types::{
             AsMapping, AsSequence, Callable, Comparable, Constructor, Hashable, IterNext,
             IterNextIterable, Iterable, PyComparisonOp,
@@ -293,7 +292,7 @@ mod _sqlite {
     #[derive(FromArgs)]
     struct ConnectArgs {
         #[pyarg(any)]
-        database: PyPathLike,
+        database: FsPath,
         #[pyarg(any, default = "5.0")]
         timeout: f64,
         #[pyarg(any, default = "0")]
@@ -828,7 +827,7 @@ mod _sqlite {
     #[pyclass(with(Constructor, Callable), flags(BASETYPE))]
     impl Connection {
         fn new(args: ConnectArgs, vm: &VirtualMachine) -> PyResult<Self> {
-            let path = args.database.into_cstring(vm)?;
+            let path = args.database.to_cstring(vm)?;
             let db = Sqlite::from(SqliteRaw::open(path.as_ptr(), args.uri, vm)?);
             let timeout = (args.timeout * 1000.0) as c_int;
             db.busy_timeout(timeout);

stdlib/src/ssl.rs

@@ -36,9 +36,9 @@ mod _ssl {
             convert::{ToPyException, ToPyObject},
             exceptions,
             function::{
-                ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, Either, OptionalArg,
+                ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath,
+                OptionalArg,
             },
-            stdlib::os::PyPathLike,
             types::Constructor,
             utils::ToCString,
             PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
@@ -726,10 +726,12 @@ mod _ssl {
                 );
             }
             let mut ctx = self.builder();
-            ctx.set_certificate_chain_file(&certfile)
+            let key_path = keyfile.map(|path| path.to_path_buf(vm)).transpose()?;
+            let cert_path = certfile.to_path_buf(vm)?;
+            ctx.set_certificate_chain_file(&cert_path)
                 .and_then(|()| {
                     ctx.set_private_key_file(
-                        keyfile.as_ref().unwrap_or(&certfile),
+                        key_path.as_ref().unwrap_or(&cert_path),
                         ssl::SslFiletype::PEM,
                     )
                 })
@@ -819,9 +821,9 @@ mod _ssl {
 
     #[derive(FromArgs)]
     struct LoadCertChainArgs {
-        certfile: PyPathLike,
+        certfile: FsPath,
         #[pyarg(any, optional)]
-        keyfile: Option<PyPathLike>,
+        keyfile: Option<FsPath>,
         #[pyarg(any, optional)]
         password: Option<Either<PyStrRef, ArgCallable>>,
     }
@@ -1308,7 +1310,8 @@ mod _ssl {
     }
 
     #[pyfunction]
-    fn _test_decode_cert(path: PyPathLike, vm: &VirtualMachine) -> PyResult {
+    fn _test_decode_cert(path: FsPath, vm: &VirtualMachine) -> PyResult {
+        let path = path.to_path_buf(vm)?;
         let pem = std::fs::read(path).map_err(|e| e.to_pyexception(vm))?;
         let x509 = X509::from_pem(&pem).map_err(|e| convert_openssl_error(vm, e))?;
         cert_to_py(vm, &x509, false)

vm/src/function/fspath.rs

@@ -0,0 +1,130 @@
+use crate::{
+    builtins::{PyBytes, PyBytesRef, PyStrRef},
+    convert::{IntoPyException, ToPyObject},
+    function::PyStr,
+    protocol::PyBuffer,
+    PyObjectRef, PyResult, TryFromObject, VirtualMachine,
+};
+use std::{ffi::OsStr, path::PathBuf};
+
+#[derive(Clone)]
+pub enum FsPath {
+    Str(PyStrRef),
+    Bytes(PyBytesRef),
+}
+
+impl FsPath {
+    // PyOS_FSPath in CPython
+    pub fn try_from(obj: PyObjectRef, check_for_nul: bool, vm: &VirtualMachine) -> PyResult<Self> {
+        let check_nul = |b: &[u8]| {
+            if !check_for_nul || memchr::memchr(b'\0', b).is_none() {
+                Ok(())
+            } else {
+                Err(crate::exceptions::cstring_error(vm))
+            }
+        };
+        let match1 = |obj: PyObjectRef| {
+            let pathlike = match_class!(match obj {
+                s @ PyStr => {
+                    check_nul(s.as_str().as_bytes())?;
+                    FsPath::Str(s)
+                }
+                b @ PyBytes => {
+                    check_nul(&b)?;
+                    FsPath::Bytes(b)
+                }
+                obj => return Ok(Err(obj)),
+            });
+            Ok(Ok(pathlike))
+        };
+        let obj = match match1(obj)? {
+            Ok(pathlike) => return Ok(pathlike),
+            Err(obj) => obj,
+        };
+        let method =
+            vm.get_method_or_type_error(obj.clone(), identifier!(vm, __fspath__), || {
+                format!(
+                    "should be string, bytes, os.PathLike or integer, not {}",
+                    obj.class().name()
+                )
+            })?;
+        let result = method.call((), vm)?;
+        match1(result)?.map_err(|result| {
+            vm.new_type_error(format!(
+                "expected {}.__fspath__() to return str or bytes, not {}",
+                obj.class().name(),
+                result.class().name(),
+            ))
+        })
+    }
+
+    pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<&OsStr> {
+        // TODO: FS encodings
+        match self {
+            FsPath::Str(s) => Ok(s.as_str().as_ref()),
+            FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm),
+        }
+    }
+
+    pub fn as_bytes(&self) -> &[u8] {
+        // TODO: FS encodings
+        match self {
+            FsPath::Str(s) => s.as_str().as_bytes(),
+            FsPath::Bytes(b) => b.as_bytes(),
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        match self {
+            FsPath::Bytes(b) => std::str::from_utf8(b).unwrap(),
+            FsPath::Str(s) => s.as_str(),
+        }
+    }
+
+    pub fn to_path_buf(&self, vm: &VirtualMachine) -> PyResult<PathBuf> {
+        let path = match self {
+            FsPath::Str(s) => PathBuf::from(s.as_str()),
+            FsPath::Bytes(b) => PathBuf::from(Self::bytes_as_osstr(b, vm)?),
+        };
+        Ok(path)
+    }
+
+    pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> {
+        std::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm))
+    }
+
+    #[cfg(windows)]
+    pub fn to_widecstring(&self, vm: &VirtualMachine) -> PyResult<widestring::WideCString> {
+        widestring::WideCString::from_os_str(self.as_os_str(vm)?)
+            .map_err(|err| err.into_pyexception(vm))
+    }
+
+    pub fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
+        rustpython_common::os::bytes_as_osstr(b)
+            .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8".to_owned()))
+    }
+}
+
+impl ToPyObject for FsPath {
+    fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
+        match self {
+            Self::Str(s) => s.into(),
+            Self::Bytes(b) => b.into(),
+        }
+    }
+}
+
+impl TryFromObject for FsPath {
+    // PyUnicode_FSDecoder in CPython
+    fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        let obj = match obj.try_to_value::<PyBuffer>(vm) {
+            Ok(buffer) => {
+                let mut bytes = vec![];
+                buffer.append_to(&mut bytes);
+                vm.ctx.new_bytes(bytes).into()
+            }
+            Err(_) => obj,
+        };
+        Self::try_from(obj, true, vm)
+    }
+}

vm/src/function/mod.rs

@@ -3,6 +3,7 @@ mod arithmetic;
 mod buffer;
 mod builtin;
 mod either;
+mod fspath;
 mod getset;
 mod number;
 mod protocol;
@@ -16,6 +17,7 @@ pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLik
 pub(self) use builtin::{BorrowedParam, OwnedParam, RefParam};
 pub use builtin::{IntoPyNativeFunc, PyNativeFunc};
 pub use either::Either;
+pub use fspath::FsPath;
 pub use getset::PySetterValue;
 pub(super) use getset::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc};
 pub use number::{ArgIndex, ArgIntoBool, ArgIntoComplex, ArgIntoFloat, ArgPrimitiveIndex, ArgSize};

vm/src/stdlib/io.rs

@@ -3545,7 +3545,7 @@ mod _io {
 
         // check file descriptor validity
         #[cfg(unix)]
-        if let Ok(crate::stdlib::os::PathOrFd::Fd(fd)) = file.clone().try_into_value(vm) {
+        if let Ok(crate::stdlib::os::OsPathOrFd::Fd(fd)) = file.clone().try_into_value(vm) {
             nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD)
                 .map_err(|_| crate::stdlib::os::errno_err(vm))?;
         }
@@ -3874,7 +3874,7 @@ mod fileio {
             } else if let Some(i) = name.payload::<crate::builtins::PyInt>() {
                 i.try_to_primitive(vm)?
             } else {
-                let path = os::PyPathLike::try_from_object(vm, name.clone())?;
+                let path = os::OsPath::try_from_object(vm, name.clone())?;
                 if !args.closefd {
                     return Err(
                         vm.new_value_error("Cannot use closefd=False with file name".to_owned())