Add ReplyIoctl::retry for variable-size ioctls

examples/ioctl.rs

@@ -167,6 +167,7 @@ impl Filesystem for FiocFS {
         _fh: FileHandle,
         _flags: IoctlFlags,
         cmd: u32,
+        _arg: u64,
         in_data: &[u8],
         _out_size: u32,
         reply: ReplyIoctl,

src/lib.rs

@@ -65,6 +65,7 @@ pub use crate::reply::ReplyDirectory;
 pub use crate::reply::ReplyDirectoryPlus;
 pub use crate::reply::ReplyEmpty;
 pub use crate::reply::ReplyEntry;
+pub use crate::reply::IoctlIovec;
 pub use crate::reply::ReplyIoctl;
 pub use crate::reply::ReplyLock;
 pub use crate::reply::ReplyLseek;
@@ -911,20 +912,30 @@ pub trait Filesystem: Send + Sync + 'static {
     }
 
     /// control device
+    ///
+    /// `arg` is the userspace pointer the ioctl was issued with
+    /// (`fuse_ioctl_in.arg`); use it together with
+    /// [`ReplyIoctl::retry`] to describe variable-size buffers that
+    /// don't fit the size encoded in `cmd` (e.g.
+    /// `BTRFS_IOC_TREE_SEARCH_V2`). On the retry pass `flags` will
+    /// contain [`IoctlFlags::FUSE_IOCTL_UNRESTRICTED`] and
+    /// `in_data` / `out_size` will reflect the iovec ranges
+    /// returned from the previous call.
     fn ioctl(
         &self,
         _req: &Request,
         ino: INodeNo,
         fh: FileHandle,
         flags: IoctlFlags,
         cmd: u32,
+        arg: u64,
         in_data: &[u8],
         out_size: u32,
         reply: ReplyIoctl,
     ) {
         warn!(
             "[Not Implemented] ioctl(ino: {ino:#x?}, fh: {fh}, flags: {flags}, \
-            cmd: {cmd}, in_data.len(): {}, out_size: {out_size})",
+            cmd: {cmd}, arg: {arg:#x}, in_data.len(): {}, out_size: {out_size})",
             in_data.len()
         );
         reply.error(Errno::ENOSYS);

src/ll/fuse_abi.rs

@@ -105,8 +105,12 @@ pub mod consts {
     // Lock flags
     pub const FUSE_LK_FLOCK: u32 = 1 << 0;
 
-    // IOCTL constant
+    // IOCTL constants
     pub const FUSE_IOCTL_MAX_IOV: u32 = 256; // maximum of in_iovecs + out_iovecs
+    pub const FUSE_IOCTL_COMPAT: u32 = 1 << 0;
+    pub const FUSE_IOCTL_UNRESTRICTED: u32 = 1 << 1;
+    pub const FUSE_IOCTL_RETRY: u32 = 1 << 2;
+    pub const FUSE_IOCTL_DIR: u32 = 1 << 4;
 
     // The read buffer is required to be at least 8k, but may be much larger
     pub const FUSE_MIN_READ_BUFFER: usize = 8192;
@@ -589,7 +593,7 @@ pub(crate) struct fuse_ioctl_in {
 }
 
 #[repr(C)]
-#[derive(Debug, KnownLayout, Immutable)]
+#[derive(Debug, IntoBytes, KnownLayout, Immutable)]
 pub(crate) struct fuse_ioctl_iovec {
     pub(crate) base: u64,
     pub(crate) len: u64,

src/ll/reply.rs

@@ -267,6 +267,28 @@ impl<'a> ResponseIoctl<'a> {
         };
         ResponseIoctl { out, iovs }
     }
+
+    /// Build a "retry with these iovecs" response. The kernel will
+    /// re-issue the ioctl with `FUSE_IOCTL_UNRESTRICTED` set and the
+    /// requested user-space ranges concatenated into `in_data` /
+    /// `out`. `payload` must be the host-endian serialisation of
+    /// `[fuse_ioctl_iovec; in_iovs] ++ [fuse_ioctl_iovec; out_iovs]`.
+    pub(crate) fn new_retry(
+        in_iovs: u32,
+        out_iovs: u32,
+        payload: &'a [IoSlice<'a>],
+    ) -> Self {
+        let out = abi::fuse_ioctl_out {
+            result: 0,
+            flags: abi::consts::FUSE_IOCTL_RETRY,
+            in_iovs,
+            out_iovs,
+        };
+        ResponseIoctl {
+            out,
+            iovs: payload,
+        }
+    }
 }
 
 impl Response for ResponseIoctl<'_> {

src/ll/request.rs

@@ -1342,9 +1342,6 @@ mod op {
         pub(crate) fn in_data(&self) -> &[u8] {
             &self.data[..self.arg.in_size as usize]
         }
-        pub(crate) fn unrestricted(&self) -> bool {
-            self.flags().contains(IoctlFlags::FUSE_IOCTL_UNRESTRICTED)
-        }
         /// The value set by the [`Open`] method. See [`FileHandle`].
         pub(crate) fn file_handle(&self) -> FileHandle {
             FileHandle(self.arg.fh)
@@ -1359,6 +1356,14 @@ mod op {
         pub(crate) fn out_size(&self) -> u32 {
             self.arg.out_size
         }
+        /// The userspace pointer the ioctl was called with. The
+        /// driver can hand portions of this address range back to
+        /// the kernel through [`crate::ReplyIoctl::retry`] when the
+        /// real input/output buffer doesn't fit the size encoded in
+        /// `cmd`.
+        pub(crate) fn arg_ptr(&self) -> u64 {
+            self.arg.arg
+        }
     }
 
     /// Poll.

src/reply.rs

@@ -655,12 +655,85 @@ impl ReplyIoctl {
             .send_ll(&ll::ResponseIoctl::new_ioctl(result, &[IoSlice::new(data)]));
     }
 
+    /// Ask the kernel to retry the ioctl with the given userspace
+    /// iovecs.
+    ///
+    /// This is the FUSE_IOCTL_RETRY mechanism: when the size encoded
+    /// in an ioctl's `cmd` is too small to describe its real input or
+    /// output buffer (typically because the struct embeds a flexible
+    /// array, or the buffer is otherwise dynamically sized), the
+    /// driver responds with the iovecs describing what it actually
+    /// wants. The kernel re-issues the ioctl with
+    /// `FUSE_IOCTL_UNRESTRICTED` set; the new request's `in_data` is
+    /// the concatenation of `in_iovs` and the new `out_size` covers
+    /// `out_iovs`.
+    ///
+    /// `in_iovs` and `out_iovs` describe ranges in the *caller's*
+    /// userspace memory. The starting pointer is typically the `arg`
+    /// value passed to [`Filesystem::ioctl`](crate::Filesystem::ioctl),
+    /// plus any offset into the struct.
+    ///
+    /// The total number of entries (in_iovs + out_iovs) must be at
+    /// most [`FUSE_IOCTL_MAX_IOV`](crate::consts::FUSE_IOCTL_MAX_IOV).
+    ///
+    /// # Panics
+    ///
+    /// Panics if `in_iovs.len() + out_iovs.len() > FUSE_IOCTL_MAX_IOV`.
+    /// The kernel rejects oversized iovec arrays at runtime, so the
+    /// panic surfaces the same bug eagerly with a clearer message.
+    pub fn retry(self, in_iovs: &[IoctlIovec], out_iovs: &[IoctlIovec]) {
+        let total = in_iovs.len() + out_iovs.len();
+        let max = crate::consts::FUSE_IOCTL_MAX_IOV as usize;
+        assert!(
+            total <= max,
+            "ReplyIoctl::retry: in_iovs ({}) + out_iovs ({}) = {} exceeds \
+             FUSE_IOCTL_MAX_IOV ({max})",
+            in_iovs.len(),
+            out_iovs.len(),
+            total,
+        );
+
+        let mut payload: Vec<u8> = Vec::with_capacity(
+            total * std::mem::size_of::<IoctlIovec>(),
+        );
+        for iov in in_iovs.iter().chain(out_iovs.iter()) {
+            payload.extend_from_slice(&iov.base.to_ne_bytes());
+            payload.extend_from_slice(&iov.len.to_ne_bytes());
+        }
+        // Bounded by FUSE_IOCTL_MAX_IOV (256) above — the casts are
+        // infallible and clippy is wrong about them.
+        #[allow(clippy::cast_possible_truncation)]
+        let in_count = in_iovs.len() as u32;
+        #[allow(clippy::cast_possible_truncation)]
+        let out_count = out_iovs.len() as u32;
+        self.reply.send_ll(&ll::ResponseIoctl::new_retry(
+            in_count,
+            out_count,
+            &[IoSlice::new(&payload)],
+        ));
+    }
+
     /// Reply to a request with the given error code
     pub fn error(self, err: Errno) {
         self.reply.error(err);
     }
 }
 
+/// Userspace memory range, used by [`ReplyIoctl::retry`] to describe
+/// the buffers the FUSE driver wants the kernel to copy on retry.
+///
+/// Mirrors the kernel's `struct fuse_ioctl_iovec`. `base` is a raw
+/// pointer-as-u64 in the caller's address space; `len` is the byte
+/// length.
+#[derive(Debug, Clone, Copy)]
+#[repr(C)]
+pub struct IoctlIovec {
+    /// User-space pointer (as a `u64`) marking the start of the range.
+    pub base: u64,
+    /// Length of the range in bytes.
+    pub len: u64,
+}
+
 ///
 /// Poll Reply
 ///
@@ -1258,6 +1331,66 @@ mod test {
         reply.data(&[0x11, 0x22, 0x33, 0x44]);
     }
 
+    #[test]
+    fn reply_ioctl() {
+        // fuse_out_header(16) + fuse_ioctl_out(16) + 4-byte payload.
+        // Header length = 36 = 0x24.
+        let sender = ReplySender::Assert(AssertSender {
+            expected: vec![
+                0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xbe, 0xad, 0xde, 0x00, 0x00,
+                0x00, 0x00, // fuse_out_header
+                0x00, 0x00, 0x00, 0x00, // result = 0
+                0x00, 0x00, 0x00, 0x00, // flags = 0
+                0x01, 0x00, 0x00, 0x00, // in_iovs = 1
+                0x01, 0x00, 0x00, 0x00, // out_iovs = 1 (one IoSlice)
+                0xde, 0xad, 0xbe, 0xef, // payload
+            ],
+        });
+        let reply: ReplyIoctl = Reply::new(ll::RequestId(0xdeadbeef), sender);
+        reply.ioctl(0, &[0xde, 0xad, 0xbe, 0xef]);
+    }
+
+    #[test]
+    fn reply_ioctl_retry() {
+        // Retry with one in_iov and one out_iov, both pointing at
+        // the same address with len=0x1000. Header length = 16 (out
+        // header) + 16 (fuse_ioctl_out) + 2*16 (two iovecs) = 64 = 0x40.
+        let sender = ReplySender::Assert(AssertSender {
+            expected: vec![
+                0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xbe, 0xad, 0xde, 0x00, 0x00,
+                0x00, 0x00, // fuse_out_header
+                0x00, 0x00, 0x00, 0x00, // result = 0
+                0x04, 0x00, 0x00, 0x00, // flags = FUSE_IOCTL_RETRY (1 << 2)
+                0x01, 0x00, 0x00, 0x00, // in_iovs = 1
+                0x01, 0x00, 0x00, 0x00, // out_iovs = 1
+                // in_iov: base = 0xdeadbeef00, len = 0x1000
+                0x00, 0xef, 0xbe, 0xad, 0xde, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, // out_iov: base = 0xdeadbeef00, len = 0x1000
+                0x00, 0xef, 0xbe, 0xad, 0xde, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00,
+            ],
+        });
+        let reply: ReplyIoctl = Reply::new(ll::RequestId(0xdeadbeef), sender);
+        let iov = IoctlIovec {
+            base: 0xdead_beef_00,
+            len: 0x1000,
+        };
+        reply.retry(&[iov], &[iov]);
+    }
+
+    #[test]
+    #[should_panic(expected = "exceeds FUSE_IOCTL_MAX_IOV")]
+    fn reply_ioctl_retry_panics_on_too_many_iovs() {
+        // Sender doesn't matter — we panic before any bytes are sent.
+        let (tx, _rx) = sync_channel::<()>(1);
+        let reply: ReplyIoctl =
+            Reply::new(ll::RequestId(0xdeadbeef), ReplySender::Sync(tx));
+        // 257 in_iovs + 0 out_iovs > FUSE_IOCTL_MAX_IOV (256).
+        let iov = IoctlIovec { base: 0, len: 0 };
+        let too_many = vec![iov; 257];
+        reply.retry(&too_many, &[]);
+    }
+
     #[test]
     fn async_reply() {
         let (tx, rx) = sync_channel::<()>(1);

src/request.rs

@@ -423,15 +423,13 @@ impl<'a> RequestWithSender<'a> {
             }
 
             ll::Operation::IoCtl(x) => {
-                if x.unrestricted() {
-                    return Err(Errno::ENOSYS);
-                }
                 filesystem.ioctl(
                     self.request_header(),
                     self.request.nodeid(),
                     x.file_handle(),
                     x.flags(),
                     x.command(),
+                    x.arg_ptr(),
                     x.in_data(),
                     x.out_size(),
                     self.reply(),