feat: 重构 DHT 爬取与可观测性

This commit is contained in:
桥下红药
2026-07-19 15:26:04 +08:00
parent bcc166c53b
commit 7c79bff593
32 changed files with 6738 additions and 1598 deletions
+34 -7
View File
@@ -34,11 +34,14 @@ async fn main() -> Result<()> {
let options = DHTOptions {
port: 12313,
metadata_timeout: 3, // ✅ 快速超时,快速失败
max_metadata_queue_size: 100000, // ✅ 大缓冲区(防止饱和)
max_metadata_worker_count: 1000, // ✅ 激进并发(最大化吞吐)
netmode: NetMode::Ipv4Only, // 网络模式:Ipv4Only(仅IPv4)、Ipv6Only(仅IPv6)、DualStack(双栈,默认)
..Default::default() // 使用默认值填充其他字段(节点队列容量等)
netmode: NetMode::Ipv4Only,
metadata: MetadataOptions {
timeout_secs: 4,
max_queue_size: 10_000,
max_worker_count: 256,
..MetadataOptions::default()
},
..Default::default()
};
// 统计计数器
@@ -83,8 +86,19 @@ async fn main() -> Result<()> {
// 设置元数据获取前的检查回调
server.on_metadata_fetch(|_hash| async move { true });
// 一个通过 gate 的 Hash 最终只会收到一次完成状态。
server.on_metadata_fetch_complete(|completion| {
log::debug!(
"Metadata complete: hash={}, status={:?}, peer_attempts={}",
completion.info_hash,
completion.status,
completion.attempts
);
});
// 启动监控任务
let count_monitor = torrent_count.clone();
let runtime_stats = server.runtime_stats();
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
let start_time = std::time::Instant::now();
@@ -93,12 +107,17 @@ async fn main() -> Result<()> {
interval.tick().await;
let success_fetch = count_monitor.load(Ordering::Relaxed);
let uptime = start_time.elapsed().as_secs();
let runtime = runtime_stats.snapshot();
// ✅ 监控:爬虫运行状态
log::info!(
"📊 [监控] 时长: {}s | 成功抓取: ✨ {}",
"📊 [监控] 时长: {}s | 成功抓取: ✨ {} | 节点: {} | Metadata: {}/{} | worker: {}",
uptime,
success_fetch
success_fetch,
runtime.node_pool_size,
runtime.metadata_queue_depth,
runtime.metadata_queue_max,
runtime.metadata_in_flight,
);
if uptime > 0 && success_fetch > 0 {
@@ -108,6 +127,14 @@ async fn main() -> Result<()> {
}
});
let shutdown_server = server.clone();
tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
log::info!("收到 Ctrl-C,正在停止 DHT Server");
shutdown_server.shutdown();
}
});
server.start().await?;
Ok(())
}