thread 'risingwave-batch-tasks' panicked at 'frozen file not found', /root/.cargo/git/checkouts/foyer-e7ae575671bf01ff/8e80c6c/foyer/src/store/read_only_file_store.rs:331:22
stack backtrace:
0: 0x55c8c29079fc - std::backtrace_rs::backtrace::libunwind::trace::h43d94404a3e24b5e
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c8c29079fc - std::backtrace_rs::backtrace::trace_unsynchronized::hb80ac700c030e86a
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c8c29079fc - std::sys_common::backtrace::_print_fmt::h6b86e24d9aefa090
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c8c29079fc - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h49d07dd1f79a975a
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c8bd191e1f - core::fmt::rt::Argument::fmt::h1864a7d8f112957b
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/fmt/rt.rs:138:9
5: 0x55c8bd191e1f - core::fmt::write::hd813644c20dd03f8
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/fmt/mod.rs:1094:21
6: 0x55c8c2902d6f - std::io::Write::write_fmt::h6fea2a18781d1e2e
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/io/mod.rs:1712:15
7: 0x55c8c2907805 - std::sys_common::backtrace::_print::h9d3d888fc0c72876
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:47:5
8: 0x55c8c2907805 - std::sys_common::backtrace::print::h04abcd5816949a30
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:34:9
9: 0x55c8c2908d67 - std::panicking::default_hook::{{closure}}::ha75ea1b8d759bad4
10: 0x55c8c2908b74 - std::panicking::default_hook::he2c557a4cfe41c90
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:288:9
11: 0x55c8c228a23b - call<(&core::panic::panic_info::PanicInfo), (dyn core::ops::function::Fn<(&core::panic::panic_info::PanicInfo), Output=()> + core::marker::Send + core::marker::Sync), alloc::alloc::Global>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/alloc/src/boxed.rs:1999:9
12: 0x55c8c228a23b - {closure#0}
at /risingwave/src/utils/runtime/src/lib.rs:119:9
13: 0x55c8c228a23b - {closure#0}<risingwave_rt::set_panic_hook::{closure_env#0}>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:234:47
14: 0x55c8c2909384 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::hcf9192dbf6344206
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/alloc/src/boxed.rs:1999:9
15: 0x55c8c2909384 - std::panicking::rust_panic_with_hook::hb4b975f86bb49bbc
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:695:13
16: 0x55c8c2909114 - std::panicking::begin_panic_handler::{{closure}}::h381a1086a75e4820
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:582:13
17: 0x55c8c2907dd6 - std::sys_common::backtrace::__rust_end_short_backtrace::h2f950c06980ecb26
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:151:18
18: 0x55c8c2908eb2 - rust_begin_unwind
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:578:5
19: 0x55c8bd18eb23 - core::panicking::panic_fmt::he9fd35b0a19637c8
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panicking.rs:67:14
20: 0x55c8bd18e983 - core::panicking::panic_display::ha29c9cb28ca203a7
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panicking.rs:150:5
21: 0x55c8bd18e983 - core::panicking::panic_str::ha5696582f17c5eba
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panicking.rs:134:5
22: 0x55c8bd18e983 - core::option::expect_failed::h2745ad5925c1c797
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/option.rs:1932:5
23: 0x55c8be065f7d - expect<&foyer::store::read_only_file_store::Frozen>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/option.rs:898:21
24: 0x55c8be065f7d - {async_block#0}<risingwave_storage::hummock::tiered_cache::SstableBlockIndex, alloc::vec::Vec<u8, alloc::alloc::Global>>
at /root/.cargo/git/checkouts/foyer-e7ae575671bf01ff/8e80c6c/foyer/src/store/read_only_file_store.rs:328:17
25: 0x55c8bdf06990 - poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<(), foyer::store::error::Error>> + core::marker::Send), alloc::alloc::Global>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/future/future.rs:125:9
26: 0x55c8bdf06990 - {async_fn#0}<risingwave_storage::hummock::tiered_cache::SstableBlockIndex, foyer::policies::tinylfu::TinyLfu<risingwave_storage::hummock::tiered_cache::SstableBlockIndex>, foyer::policies::tinylfu::Handle<risingwave_storage::hummock::tiered_cache::SstableBlockIndex>, alloc::vec::Vec<u8, alloc::alloc::Global>, foyer::store::read_only_file_store::ReadOnlyFileStore<risingwave_storage::hummock::tiered_cache::SstableBlockIndex, alloc::vec::Vec<u8, alloc::alloc::Global>>>
at /root/.cargo/git/checkouts/foyer-e7ae575671bf01ff/8e80c6c/foyer/src/container.rs:236:38
27: 0x55c8bdf06990 - {async_fn#0}<risingwave_storage::hummock::tiered_cache::SstableBlockIndex, foyer::policies::tinylfu::TinyLfu<risingwave_storage::hummock::tiered_cache::SstableBlockIndex>, foyer::policies::tinylfu::Handle<risingwave_storage::hummock::tiered_cache::SstableBlockIndex>, alloc::vec::Vec<u8, alloc::alloc::Global>, foyer::store::read_only_file_store::ReadOnlyFileStore<risingwave_storage::hummock::tiered_cache::SstableBlockIndex, alloc::vec::Vec<u8, alloc::alloc::Global>>>
at /root/.cargo/git/checkouts/foyer-e7ae575671bf01ff/8e80c6c/foyer/src/container.rs:142:32
28: 0x55c8bdf06990 - {async_fn#0}
at /risingwave/src/storage/src/hummock/tiered_cache.rs:120:18
29: 0x55c8bdf06990 - {async_block#0}
at /risingwave/src/storage/src/hummock/sstable_store.rs:206:69
30: 0x55c8bdf5cc59 - {async_block#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>
at /risingwave/src/storage/src/hummock/block_cache.rs:205:35
31: 0x55c8bdf5cc59 - {async_block#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>
at /risingwave/src/common/src/cache.rs:967:55
32: 0x55c8bdf5cc59 - poll<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
33: 0x55c8bdf87987 - {closure#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/core.rs:223:17
34: 0x55c8bdf87987 - with_mut<tokio::runtime::task::core::Stage<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>>, core::task::poll::Poll<core::result::Result<risingwave_common::cache::CacheableEntry<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>>, risingwave_storage::hummock::error::HummockError>>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/loom/std/unsafe_cell.rs:14:9
35: 0x55c8bdf48d74 - poll<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/core.rs:212:13
36: 0x55c8be04585a - {closure#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:476:19
37: 0x55c8be04585a - call_once<core::task::poll::Poll<core::result::Result<risingwave_common::cache::CacheableEntry<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>>, risingwave_storage::hummock::error::HummockError>>, tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panic/unwind_safe.rs:271:9
38: 0x55c8be04585a - do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>>, core::task::poll::Poll<core::result::Result<risingwave_common::cache::CacheableEntry<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>>, risingwave_storage::hummock::error::HummockError>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:485:40
39: 0x55c8be04585a - try<core::task::poll::Poll<core::result::Result<risingwave_common::cache::CacheableEntry<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>>, risingwave_storage::hummock::error::HummockError>>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:449:19
40: 0x55c8be04585a - catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>>, core::task::poll::Poll<core::result::Result<risingwave_common::cache::CacheableEntry<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>>, risingwave_storage::hummock::error::HummockError>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panic.rs:142:14
41: 0x55c8be04585a - poll_future<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:464:18
42: 0x55c8be04585a - poll_inner<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:198:27
43: 0x55c8be04585a - poll<tracing::instrument::Instrumented<risingwave_common::cache::{impl#14}::lookup_with_request_dedup::{async_block_env#0}<(u64, u64), alloc::boxed::Box<risingwave_storage::hummock::sstable::block::Block, alloc::alloc::Global>, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>, risingwave_storage::hummock::error::HummockError, risingwave_storage::hummock::block_cache::{impl#5}::get_or_insert_with::{closure#0}::{async_block_env#0}<risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure_env#0}, risingwave_storage::hummock::sstable_store::{impl#1}::get_block_response::{async_fn#0}::{closure#0}::{async_block_env#0}>>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:152:15
44: 0x55c8c29a516b - poll
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/raw.rs:200:18
45: 0x55c8c29a516b - run<alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/mod.rs:394:9
46: 0x55c8c29a516b - {closure#0}
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:464:18
47: 0x55c8c29a516b - with_budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/coop.rs:107:5
48: 0x55c8c29a516b - budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/coop.rs:73:5
49: 0x55c8c29a516b - run_task
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:463:9
50: 0x55c8c29a47c3 - run
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:426:24
51: 0x55c8c2997f2d - {closure#0}
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:406:17
52: 0x55c8c2997f2d - set<tokio::runtime::scheduler::multi_thread::worker::Context, tokio::runtime::scheduler::multi_thread::worker::run::{closure_env#0}, ()>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/macros/scoped_tls.rs:61:9
53: 0x55c8c29a4238 - run
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:403:5
54: 0x55c8c2986dfc - {closure#0}
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/scheduler/multi_thread/worker.rs:365:45
55: 0x55c8c2986dfc - poll<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}, ()>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/blocking/task.rs:42:21
56: 0x55c8c2986dfc - poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
57: 0x55c8c298c29d - {closure#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/core.rs:223:17
58: 0x55c8c298c29d - with_mut<tokio::runtime::task::core::Stage<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/loom/std/unsafe_cell.rs:14:9
59: 0x55c8c298c29d - poll<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/core.rs:212:13
60: 0x55c8c296ed2f - {closure#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:476:19
61: 0x55c8c296ed2f - call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panic/unwind_safe.rs:271:9
62: 0x55c8c296ed2f - do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:485:40
63: 0x55c8c296ed2f - try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:449:19
64: 0x55c8c296ed2f - catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panic.rs:142:14
65: 0x55c8c296ed2f - poll_future<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:464:18
66: 0x55c8c296ed2f - poll_inner<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:198:27
67: 0x55c8c296ed2f - poll<tracing::instrument::Instrumented<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/harness.rs:152:15
68: 0x55c8c296d910 - poll
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/raw.rs:200:18
69: 0x55c8c296d910 - run<tokio::runtime::blocking::schedule::BlockingSchedule>
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/task/mod.rs:431:9
70: 0x55c8c296d910 - run
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/blocking/pool.rs:159:9
71: 0x55c8c296d910 - run
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/blocking/pool.rs:513:17
72: 0x55c8c2995e45 - {closure#0}
at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.28.0/src/runtime/blocking/pool.rs:471:13
73: 0x55c8c2995e45 - __rust_begin_short_backtrace<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys_common/backtrace.rs:135:18
74: 0x55c8c297a692 - {closure#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/thread/mod.rs:529:17
75: 0x55c8c297a692 - call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/panic/unwind_safe.rs:271:9
76: 0x55c8c297a692 - do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:485:40
77: 0x55c8c297a692 - try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panicking.rs:449:19
78: 0x55c8c297a692 - catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/panic.rs:142:14
79: 0x55c8c297a692 - {closure#1}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/thread/mod.rs:528:30
80: 0x55c8c297a692 - call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#1}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>, ()>
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/core/src/ops/function.rs:250:5
81: 0x55c8c290f035 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h05bc36bea856dce6
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/alloc/src/boxed.rs:1985:9
82: 0x55c8c290f035 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h76ae250201c440c9
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/alloc/src/boxed.rs:1985:9
83: 0x55c8c290f035 - std::sys::unix::thread::Thread::new::thread_start::h032990071e345e22
at /rustc/c373194cb6d882dc455a588bcc29c92a96b50252/library/std/src/sys/unix/thread.rs:108:17
84: 0x7f8b5a2c3b43 - <unknown>
85: 0x7f8b5a355a00 - <unknown>
86: 0x0 - <unknown>