Fix leaking fiber stacks on OOM

crates/fiber/src/lib.rs

@@ -41,6 +41,12 @@ cfg_if::cfg_if! {
 /// Represents an execution stack to use for a fiber.
 pub struct FiberStack(imp::FiberStack);
 
+impl core::fmt::Debug for FiberStack {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.debug_struct("FiberStack").finish_non_exhaustive()
+    }
+}
+
 fn _assert_send_sync() {
     fn _assert_send<T: Send>() {}
     fn _assert_sync<T: Sync>() {}
@@ -171,11 +177,17 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> {
     /// This function returns a `Fiber` which, when resumed, will execute `func`
     /// to completion. When desired the `func` can suspend itself via
     /// `Fiber::suspend`.
+    /// On error the provided `stack` is handed back to the caller (paired with
+    /// the error) so that it can be deallocated rather than leaked; the stack is
+    /// only consumed by this `Fiber` on success.
     pub fn new(
         stack: FiberStack,
         func: impl FnOnce(Resume, &mut Suspend<Resume, Yield, Return>) -> Return + 'a,
-    ) -> Result<Self> {
-        let inner = imp::Fiber::new(&stack.0, func)?;
+    ) -> Result<Self, (Error, FiberStack)> {
+        let inner = match imp::Fiber::new(&stack.0, func) {
+            Ok(inner) => inner,
+            Err(e) => return Err((e, stack)),
+        };
 
         Ok(Self {
             stack: Some(stack),

crates/fiber/src/windows.rs

@@ -6,6 +6,7 @@ use std::io;
 use std::mem::needs_drop;
 use std::ops::Range;
 use std::ptr;
+use wasmtime_environ::prelude::*;
 use windows_sys::Win32::Foundation::*;
 use windows_sys::Win32::System::Threading::*;
 
@@ -102,13 +103,13 @@ where
 }
 
 impl Fiber {
-    pub fn new<F, A, B, C>(stack: &FiberStack, func: F) -> io::Result<Self>
+    pub fn new<F, A, B, C>(stack: &FiberStack, func: F) -> Result<Self>
     where
         F: FnOnce(A, &mut super::Suspend<A, B, C>) -> C,
     {
         unsafe {
             let state = Box::new(StartState {
-                initial_closure: Cell::new(Box::into_raw(Box::new(func)).cast()),
+                initial_closure: Cell::new(Box::into_raw(try_new(func)?).cast()),
                 parent: Cell::new(ptr::null_mut()),
                 result_location: Cell::new(ptr::null()),
             });
@@ -123,7 +124,7 @@ impl Fiber {
 
             if fiber.is_null() {
                 drop(Box::from_raw(state.initial_closure.get().cast::<F>()));
-                return Err(io::Error::last_os_error());
+                return Err(io::Error::last_os_error().into());
             }
 
             Ok(Self { fiber, state })

crates/fuzzing/tests/oom/fiber_stack.rs

@@ -0,0 +1,35 @@
+#![cfg(arc_try_new)]
+
+use wasmtime::{
+    Config, Engine, InstanceAllocationStrategy, Linker, Module, PoolingAllocationConfig, Result,
+    Store,
+};
+use wasmtime_fuzzing::oom::OomTest;
+
+#[tokio::test]
+async fn pooling_allocator_fiber_stack_slot_leak_on_oom() -> Result<()> {
+    let mut pool = PoolingAllocationConfig::default();
+    pool.total_stacks(1);
+    pool.total_memories(10);
+    pool.total_tables(10);
+    pool.total_core_instances(10);
+
+    let mut config = Config::new();
+    config.concurrency_support(false);
+    config.allocation_strategy(InstanceAllocationStrategy::Pooling(pool));
+    let engine = Engine::new(&config)?;
+    let module = Module::new(&engine, r#"(module (func (export "f")))"#)?;
+    let linker = Linker::<()>::new(&engine);
+    let instance_pre = linker.instantiate_pre(&module)?;
+
+    OomTest::new()
+        .allow_alloc_after_oom(true)
+        .test_async(|| async {
+            let mut store = Store::try_new(&engine, ())?;
+            let instance = instance_pre.instantiate_async(&mut store).await?;
+            let f = instance.get_typed_func::<(), ()>(&mut store, "f")?;
+            f.call_async(&mut store, ()).await?;
+            Ok(())
+        })
+        .await
+}

crates/fuzzing/tests/oom/main.rs

@@ -14,6 +14,7 @@ mod config;
 mod engine;
 mod entity_set;
 mod error;
+mod fiber_stack;
 mod func;
 mod func_type;
 mod fuzz;

crates/wasmtime/src/runtime/fiber.rs

@@ -836,7 +836,16 @@ where
         let reset = ResetCurrentPointersToNull(store_ref);
 
         fun(reset.0)
-    })?;
+    });
+    let fiber = match fiber {
+        Ok(fiber) => fiber,
+        Err((e, stack)) => {
+            unsafe {
+                engine.allocator().deallocate_fiber_stack(stack);
+            }
+            return Err(e);
+        }
+    };
     Ok(StoreFiber {
         state: Some(
             FiberResumeState {