Rust stackoverflow - find large variables on the stack

Question

I'm currently experiencing a very perculiar stack overflow.

fn generate_all_possible_attacks_for(
    all_relevant_moves: &U64PerSquare,
    number_of_all_relevant_moves: &U64PerSquare,
    magic_numbers: &U64PerSquare,
    calculate_attacks_for: &dyn Fn(&dyn BoardPos, u64) -> u64,
) -> Box<[U64PerSquare; 4096]> {
    let mut all_attacks = Box::new([U64PerSquare::default(); 4096]);
    //...
}

I can add a break point on the function itself, but just after (like when I put a break point on the first line in the function, or add a print) is never reached and the program experiences a stack overflow.

Exception thrown at 0x00007FF69C048067 in chess_ai.exe: 0xC00000FD: Stack overflow (parameters: 0x0000000000000001, 0x000000E296606000).

thread 'main' has overflowed its stack

I'm suspecting a couple of too large variables are placed on the stack, since the stack trace itself is fairly short:

chess_ai.exe!__chkstk() Line 109 (d:\a01\_work\6\s\src\vctools\crt\vcstartup\src\misc\amd64\chkstk.asm:109)
chess_ai.exe!chess_logic::magic_bit_board::generate_all_possible_attacks_for(unsigned __int64[64] * all_relevant_moves, unsigned __int64[64] * number_of_all_relevant_moves, unsigned __int64[64] * magic_numbers, ref$<dyn$<core::ops::function::Fn<tuple$<ref$<dyn$<chess_logic::square::BoardPos>>,u64>,assoc$<Output,u64>>>>) Line 302 (c:\Users\elias\Documents\Projects\chess_ai\chess_logic\src\magic_bit_board.rs:302)
chess_ai.exe!chess_logic::magic_bit_board::generate_all_possible_rook_attacks() Line 294 (c:\Users\elias\Documents\Projects\chess_ai\chess_logic\src\magic_bit_board.rs:294)
chess_ai.exe!core::ops::function::FnOnce::call_once<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)(),tuple$<>>(unsigned __int64[4096][64] *(*)()) Line 227 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\core\src\ops\function.rs:227)
chess_ai.exe!once_cell::sync::impl$11::force::closure$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>(once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>) Line 1212 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1212)
chess_ai.exe!once_cell::sync::impl$6::get_or_init::closure$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>(once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>) Line 1023 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1023)
chess_ai.exe!once_cell::imp::impl$4::initialize::closure$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>,enum$<once_cell::sync::impl$6::get_or_init::Void>>(once_cell::imp::impl$4::initialize::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>,enum$<once_cell::sync::impl$6::get_or_init::Void>> *) Line 85 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\imp_std.rs:85)
chess_ai.exe!core::ops::function::impls::impl$3::call_mut<tuple$<>,dyn$<core::ops::function::FnMut<tuple$<>,assoc$<Output,bool>>>>(ref_mut$<dyn$<core::ops::function::FnMut<tuple$<>,assoc$<Output,bool>>>> * self) Line 269 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\core\src\ops\function.rs:269)
chess_ai.exe!once_cell::imp::initialize_or_wait(core::sync::atomic::AtomicUsize * queue, enum$<core::option::Option<ref_mut$<dyn$<core::ops::function::FnMut<tuple$<>,assoc$<Output,bool>>>>>, 1, 18446744073709551615, Some>) Line 213 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\imp_std.rs:213)
chess_ai.exe!once_cell::imp::OnceCell<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>>::initialize<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>,enum$<once_cell::sync::impl$6::get_or_init::Void>>(once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>> self) Line 81 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\imp_std.rs:81)
chess_ai.exe!once_cell::sync::OnceCell<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>>::get_or_try_init<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>,enum$<once_cell::sync::impl$6::get_or_init::Void>>(once_cell::sync::impl$6::get_or_init::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>> self) Line 1063 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1063)
chess_ai.exe!once_cell::sync::OnceCell<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>>::get_or_init<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>>(once_cell::sync::impl$11::force::closure_env$0<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()> self) Line 1023 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1023)
chess_ai.exe!once_cell::sync::Lazy<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>::force<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>() Line 1211 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1211)
chess_ai.exe!once_cell::sync::impl$12::deref<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()>(once_cell::sync::Lazy<alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global>,alloc::boxed::Box<array$<array$<u64,64>,4096>,alloc::alloc::Global> (*)()> * self) Line 1221 (c:\Users\elias\.cargo\registry\src\github.com-1ecc6299db9ec823\once_cell-1.12.0\src\lib.rs:1221)
chess_ai.exe!chess_logic::magic_bit_board::get_rook_attacks_for(ref$<dyn$<chess_logic::square::BoardPos>> blockers, unsigned __int64) Line 48 (c:\Users\elias\Documents\Projects\chess_ai\chess_logic\src\magic_bit_board.rs:48)
chess_ai.exe!chess_ai::main() Line 19 (c:\Users\elias\Documents\Projects\chess_ai\src\main.rs:19)
chess_ai.exe!core::ops::function::FnOnce::call_once<void (*)(),tuple$<>>(void(*)()) Line 227 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\core\src\ops\function.rs:227)
chess_ai.exe!std::sys_common::backtrace::__rust_begin_short_backtrace<void (*)(),tuple$<>>(void(*)() f) Line 125 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\std\src\sys_common\backtrace.rs:125)
chess_ai.exe!std::rt::lang_start::closure$0<tuple$<>>(std::rt::lang_start::closure_env$0<tuple$<>> *) Line 145 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\std\src\rt.rs:145)
[Inline Frame] chess_ai.exe!core::ops::function::impls::impl$2::call_once() Line 259 (c:\rustc\fe5b13d681f25ee6474be29d748c65adcd91f69e\library\core\src\ops\function.rs:259)

Is there any way to figure out what takes up space in the stack?

When trying to debug with gdb (which apparently completely kicks the bucket) (I'm also really inexperienced with gdb):

GNU gdb (GDB) 8.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-w64-mingw32".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from .\target\debug\chess_ai.exe...(no debugging symbols found)...done.
(gdb) run
Starting program: C:\Users\elias\Documents\Projects\chess_ai\target\debug\chess_ai.exe 
[New Thread 29132.0x4d98]
[New Thread 29132.0x954]
[New Thread 29132.0x793c]
[New Thread 29132.0x603c]
Thread 1 received signal SIGSEGV, Segmentation fault.
0x00007ff69c048067 in ?? ()
(gdb) bt
#0  0x00007ff69c048067 in ?? ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)

The program actually works perfectly fine on Linux (the crash occurs on windows). I was told that the stack is considerably larger on Linux. So if there is any gdb trick to find large variables, I could do that on Linux.

Answer 1

A boxed array is basically a vector with a fixed size, but with one caveat: when you create it using Box::new([...]) , the array is created on the stack, then moved to the heap by Box::new() , whereas a vector's element data is created directly on the heap. If you find yourself creating huge boxed arrays like this one then you might be better off using vec![U64PerSquare::default(); 4096] vec![U64PerSquare::default(); 4096] instead, which allocates directly on the heap. The problem is most likely that you have multiple nested levels of function calls that create these boxed arrays -- that or 4096 * size_of::<U64PerSquare>() is larger than the stack already.

If you really do want the fixed-size type, there is a three-step process that gets you there without a stack allocation:

1. Create the Vec<T> .
2. Convert it to a Box<[T]> , which is an infallible operation.
3. Convert that to a Box<[T; N]> Box<[T; N]> , which fails if the length of the boxed slice is not exactly equal to N .

In your case, that would look something like:

fn new_board() -> Box<[U64PerSquare; 4096]> {
    vec![Default::default(); 4096]
        .into_boxed_slice()
        .try_into()
        .unwrap()
}

You can generalize this pattern with generics:

fn new_boxed_array<T: Clone + Default, const N: usize>() -> Box<[T; N]> {
    vec![Default::default(); N]
        .into_boxed_slice()
        .try_into()
        // Mapping the error to a string prevents the requirement T:Debug,
        // and This Should Never Fail(TM) anyway.
        .map_err(|_| "conversion from slice to array failed")
        .unwrap()
}