Report the full invalid UTF-8 span in the converted decode errors

The conversions hard-coded the decode-error span as valid_up_to() + 1,
which under-reports multi-byte invalid sequences. Derive the span from
the Utf8Error instead: valid_up_to() + error_len(), or to the end of the
input for a truncated sequence (error_len() == None), matching CPython.
Centralize this in a new_unicode_decode_error_utf8 helper so the offset
logic lives in one place.

Assisted-by: Claude Code:claude-fable-5

Author: devyubin

crates/stdlib/src/csv.rs

@@ -65,13 +65,7 @@ mod _csv {
         bytes: &[u8],
         err: core::str::Utf8Error,
     ) -> PyBaseExceptionRef {
-        vm.new_unicode_decode_error_real(
-            vm.ctx.new_str("utf-8"),
-            vm.ctx.new_bytes(bytes.to_vec()),
-            err.valid_up_to(),
-            err.valid_up_to() + 1,
-            vm.ctx.new_str("csv not utf8"),
-        )
+        vm.new_unicode_decode_error_utf8(bytes, err, "csv not utf8")
     }
 
     #[pyattr]

crates/stdlib/src/socket.rs

@@ -2613,13 +2613,7 @@ mod _socket {
             Some(ArgStrOrBytesLike::Buf(b)) => {
                 let bytes = b.borrow_buf();
                 let host_str = core::str::from_utf8(&bytes).map_err(|e| {
-                    vm.new_unicode_decode_error_real(
-                        vm.ctx.new_str("utf-8"),
-                        vm.ctx.new_bytes(bytes.to_vec()),
-                        e.valid_up_to(),
-                        e.valid_up_to() + 1,
-                        vm.ctx.new_str("host bytes is not utf8"),
-                    )
+                    vm.new_unicode_decode_error_utf8(&bytes, e, "host bytes is not utf8")
                 })?;
                 Some(host_str.to_owned())
             }
@@ -2655,13 +2649,7 @@ mod _socket {
                         let bytes = b.borrow_buf();
                         core::str::from_utf8(&bytes)
                             .map_err(|e| {
-                                vm.new_unicode_decode_error_real(
-                                    vm.ctx.new_str("utf-8"),
-                                    vm.ctx.new_bytes(bytes.to_vec()),
-                                    e.valid_up_to(),
-                                    e.valid_up_to() + 1,
-                                    vm.ctx.new_str("port is not utf8"),
-                                )
+                                vm.new_unicode_decode_error_utf8(&bytes, e, "port is not utf8")
                             })?
                             .to_owned()
                     }

crates/vm/src/function/fspath.rs

@@ -125,15 +125,8 @@ impl FsPath {
     }
 
     pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
-        rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
-            vm.new_unicode_decode_error_real(
-                vm.ctx.new_str("utf-8"),
-                vm.ctx.new_bytes(b.to_vec()),
-                e.valid_up_to(),
-                e.valid_up_to() + 1,
-                vm.ctx.new_str("can't decode path for utf-8"),
-            )
-        })
+        rustpython_host_env::os::bytes_as_os_str(b)
+            .map_err(|e| vm.new_unicode_decode_error_utf8(b, e, "can't decode path for utf-8"))
     }
 }
 

crates/vm/src/stdlib/os.rs

@@ -130,15 +130,8 @@ pub(super) struct FollowSymlinks(
 
 #[cfg(not(windows))]
 fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
-    rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
-        vm.new_unicode_decode_error_real(
-            vm.ctx.new_str("utf-8"),
-            vm.ctx.new_bytes(b.to_vec()),
-            e.valid_up_to(),
-            e.valid_up_to() + 1,
-            vm.ctx.new_str("can't decode path for utf-8"),
-        )
-    })
+    rustpython_host_env::os::bytes_as_os_str(b)
+        .map_err(|e| vm.new_unicode_decode_error_utf8(b, e, "can't decode path for utf-8"))
 }
 
 pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {

crates/vm/src/stdlib/posix.rs

@@ -1732,13 +1732,7 @@ pub mod module {
             return Err(vm.new_os_error("unable to determine login name"));
         };
         login.to_str().map(|s| s.to_owned()).map_err(|e| {
-            vm.new_unicode_decode_error_real(
-                vm.ctx.new_str("utf-8"),
-                vm.ctx.new_bytes(login.as_bytes().to_vec()),
-                e.valid_up_to(),
-                e.valid_up_to() + 1,
-                vm.ctx.new_str("unable to decode login name"),
-            )
+            vm.new_unicode_decode_error_utf8(login.as_bytes(), e, "unable to decode login name")
         })
     }
 

crates/vm/src/vm/vm_new.rs

@@ -529,6 +529,26 @@ impl VirtualMachine {
         exc
     }
 
+    /// Build a `UnicodeDecodeError` for a failed UTF-8 decode of `bytes`, taking
+    /// the invalid span from `err`: the whole invalid sequence, or up to the end
+    /// of the input for a truncated one (`error_len() == None`).
+    pub fn new_unicode_decode_error_utf8(
+        &self,
+        bytes: &[u8],
+        err: core::str::Utf8Error,
+        reason: &str,
+    ) -> PyBaseExceptionRef {
+        let start = err.valid_up_to();
+        let end = err.error_len().map_or(bytes.len(), |len| start + len);
+        self.new_unicode_decode_error_real(
+            self.ctx.new_str("utf-8"),
+            self.ctx.new_bytes(bytes.to_vec()),
+            start,
+            end,
+            self.ctx.new_str(reason),
+        )
+    }
+
     pub fn new_unicode_encode_error_real(
         &self,
         encoding: PyStrRef,