一个好的问题描述有利于社区小伙伴更快帮你定位到问题,高效解决你的问题
【TiDB 使用环境】生产环境
【TiDB 版本】v7.5.2
【部署方式】物理机部署
【操作系统/CPU 架构/芯片详情】centos7.5 x86
【遇到的问题:问题现象及影响】tikv节点异常重启。日志:
[2026/01/16 21:02:56.251 +08:00] [FATAL] [lib.rs:510] ["Invalid Iterator: Error(Request(message: \"Engine Engine(Status { code: IoError, sub_code: Non
e, sev: NoError, state: \\\"Corruption: block checksum mismatch: stored = 2324967102, computed = 1740378098, type = 1 in /mnt/ssdraid0/2/tidb70/tidb-
data/tikv/db/496130.sst offset 295682 size 9452\\\" })\"))"] [backtrace=" 0: tikv_util::set_panic_hook::{{closure}}\n at /workspace/sour
ce/tikv/components/tikv_util/src/lib.rs:509:18\n 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\n at /root/.rustup/t
oolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2032:9\n std::panicking::rust_panic_with_ho
ok\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:692:13\n
2: std::panicking::begin_panic_handler::{{closure}}\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/
src/rust/library/std/src/panicking.rs:579:13\n 3: std::sys_common::backtrace::__rust_end_short_backtrace\n at /root/.rustup/toolchains/n
ightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:137:18\n 4: rust_begin_unwind\n
at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:575:5\n 5: core::panicking
::panic_fmt\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:6
5:14\n 6: core::result::unwrap_failed\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/lib
rary/core/src/result.rs:1791:5\n 7: core::result::Result<T,E>::expect\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-li
nux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:1070:23\n tikv_kv::cursor::Cursor<I>::next\n at /workspace/source/tikv/compon
ents/tikv_kv/src/cursor.rs:370:9\n 8: tikv::storage::mvcc::reader::scanner::forward::Cursors<S>::move_write_cursor_to_next_user_key\n at
/workspace/source/tikv/src/storage/mvcc/reader/scanner/forward.rs:80:17\n 9: <tikv::storage::mvcc::reader::scanner::forward::LatestEntryPolicy as t
ikv::storage::mvcc::reader::scanner::forward::ScanPolicy<S>>::handle_write\n at /workspace/source/tikv/src/storage/mvcc/reader/scanner/for
ward.rs:622:9\n tikv::storage::mvcc::reader::scanner::forward::ForwardScanner<S,P>::read_next\n at /workspace/source/tikv/src/storage
/mvcc/reader/scanner/forward.rs:290:56\n <tikv::storage::mvcc::reader::scanner::forward::ForwardScanner<S,P> as tikv::storage::txn::store::TxnEnt
ryScanner>::next_entry\n at /workspace/source/tikv/src/storage/mvcc/reader/scanner/forward.rs:877:12\n tikv::storage::txn::store::Txn
EntryScanner::scan_entries\n at /workspace/source/tikv/src/storage/txn/store.rs:120:19\n 10: backup::endpoint::BackupRange::backup::{{clo
sure}}\n at /workspace/source/tikv/components/backup/src/endpoint.rs:406:29\n 11: <core::future::from_generator::GenFuture<T> as core::fu
ture::future::Future>::poll\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/sr
c/future/mod.rs:91:19\n resource_control::future::with_resource_limiter::{{closure}}\n at /workspace/source/tikv/components/resource_
control/src/future.rs:240:10\n <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n at /root/.rustup/
toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:91:19\n backup::endpoint::Endpoint<E,R
>::spawn_backup_worker::{{closure}}\n at /workspace/source/tikv/components/backup/src/endpoint.rs:1034:29\n 12: <core::future::from_gener
ator::GenFuture<T> as core::future::future::Future>::poll\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/ru
stlib/src/rust/library/core/src/future/mod.rs:91:19\n backup::utils::DaemonRuntime::spawn::{{closure}}\n at /workspace/source/tikv/co
mponents/backup/src/utils.rs:29:14\n <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n at /root/.r
ustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:91:19\n tokio::runtime::task::co
re::CoreStage<T>::poll::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/run
time/task/core.rs:184:17\n tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n at /workspace/.cargo/registry/src/mirrors.tuna.ts
inghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/loom/std/unsafe_cell.rs:14:9\n tokio::runtime::task::core::CoreStage<T>::poll\n at /w
orkspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:174:13\n 13: tokio::runtime::task::h
arness::poll_future::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtim
e/task/harness.rs:480:19\n <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n at /root/.
rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::panicking::
try::do_call\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:4
83:40\n std::panicking::try\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/st
d/src/panicking.rs:447:19\n std::panic::catch_unwind\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/ru
stlib/src/rust/library/std/src/panic.rs:137:14\n tokio::runtime::task::harness::poll_future\n at /workspace/.cargo/registry/src/mirro
rs.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:468:18\n tokio::runtime::task::harness::Harness<T,S>::poll_inne
r\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:104:27\n
tokio::runtime::task::harness::Harness<T,S>::poll\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio
-1.21.2/src/runtime/task/harness.rs:57:15\n tokio::runtime::task::raw::poll\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua
.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:194:5\n 14: tokio::runtime::task::raw::RawTask::poll\n at /workspace/.cargo
/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:134:18\n tokio::runtime::task::LocalNotified<S>:
:run\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/mod.rs:385:9\n t
okio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.e
du.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:421:13\n tokio::coop::with_budget::{{closure}}\n at
/workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:102:9\n std::thread::local::LocalKey<T>::tr
y_with\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:446:
16\n std::thread::local::LocalKey<T>::with\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/
rust/library/std/src/thread/local.rs:422:9\n tokio::coop::with_budget\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.c
n-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:95:5\n tokio::coop::budget\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.
cn-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:72:5\n tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n at /workspac
e/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:420:9\n 15: tokio::runt
ime::scheduler::multi_thread::worker::Context::run\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio
-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:387:24\n tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}\n at
/workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:372:17\n t
okio::macros::scoped_tls::ScopedKey<T>::set\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2
/src/macros/scoped_tls.rs:61:9\n tokio::runtime::scheduler::multi_thread::worker::run\n at /workspace/.cargo/registry/src/mirrors.tun
a.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:369:5\n tokio::runtime::scheduler::multi_thread::wor
ker::Launch::launch::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtim
e/scheduler/multi_thread/worker.rs:348:45\n <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n
at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/task.rs:42:21\n tokio::runtime
::task::core::CoreStage<T>::poll::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21
.2/src/runtime/task/core.rs:184:17\n tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n at /workspace/.cargo/registry/src/mirro
rs.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/loom/std/unsafe_cell.rs:14:9\n tokio::runtime::task::core::CoreStage<T>::poll\n
at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:174:13\n tokio::runtim
e::task::harness::poll_future::{{closure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/
src/runtime/task/harness.rs:480:19\n <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n
at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::p
anicking::try::do_call\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/pani
cking.rs:483:40\n std::panicking::try\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/
library/std/src/panicking.rs:447:19\n std::panic::catch_unwind\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-
gnu/lib/rustlib/src/rust/library/std/src/panic.rs:137:14\n tokio::runtime::task::harness::poll_future\n at /workspace/.cargo/registry
/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:468:18\n tokio::runtime::task::harness::Harness<T,S>:
:poll_inner\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:104
:27\n tokio::runtime::task::harness::Harness<T,S>::poll\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42
cdbd/tokio-1.21.2/src/runtime/task/harness.rs:57:15\n tokio::runtime::task::raw::poll\n at /workspace/.cargo/registry/src/mirrors.tun
a.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:194:5\n 16: tokio::runtime::task::raw::RawTask::poll\n at /worksp
ace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:134:18\n tokio::runtime::task::Unowned
Task<S>::run\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/mod.rs:422:9\
n tokio::runtime::blocking::pool::Task::run\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1
.21.2/src/runtime/blocking/pool.rs:111:9\n tokio::runtime::blocking::pool::Inner::run\n at /workspace/.cargo/registry/src/mirrors.tun
a.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/pool.rs:346:17\n tokio::runtime::blocking::pool::Spawner::spawn_thread::{{cl
osure}}\n at /workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/pool.rs:321:13
\n std::sys_common::backtrace::__rust_begin_short_backtrace\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu
/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:121:18\n 17: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n
at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:551:17\n <core::p
anic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n at /root/.rustup/toolchains/nightly-2022-11-15-x86
_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::panicking::try::do_call\n at /root/.rus
tup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:483:40\n std::panicking::try\n
at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:447:19\n std::pa
nic::catch_unwind\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:
137:14\n std::thread::Builder::spawn_unchecked_::{{closure}}\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gn
u/lib/rustlib/src/rust/library/std/src/thread/mod.rs:550:30\n core::ops::function::FnOnce::call_once{{vtable.shim}}\n at /root/.rustu
p/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:513:5\n 18: <alloc::boxed::Box<F,A> as
core::ops::function::FnOnce<Args>>::call_once\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/r
ust/library/alloc/src/boxed.rs:2000:9\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n at /root/.rustup/t
oolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2000:9\n std::sys::unix::thread::Thread::ne
w::thread_start\n at /root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/th
read.rs:108:17\n 19: start_thread\n 20: clone\n"] [location=/workspace/source/tikv/components/tikv_kv/src/cursor.rs:370] [thread_name=bkwkr] [thread
_id=1702]