feat: 重组项目结构并完善运行管理
This commit is contained in:
chuan
@@ -4,9 +4,11 @@
|
||||
|
||||
本项目用于持续或间歇地从 BitTorrent DHT 网络发现 infohash 获取元数据并提供本地全文搜索和高性能过滤能力
|
||||
|
||||
基础 DHT 协议和抓取能力保留在 `dht-crawler` 中
|
||||
基础 DHT 协议和抓取能力保留在 `src/crawler` 中
|
||||
|
||||
面向最终用户运行的服务代码统一放在 `dht-search` 中
|
||||
面向最终用户运行的服务代码统一放在 `src/search` 中
|
||||
|
||||
Web 前端代码统一放在 `src/web` 中
|
||||
|
||||
## 技术方案
|
||||
|
||||
@@ -17,6 +19,7 @@
|
||||
- BLAKE3 负责计算规范化内容结构指纹
|
||||
- Serde 负责配置领域对象和接口数据的序列化
|
||||
- Tracing 负责结构化日志和故障定位
|
||||
- SQLite 负责有保留上限的运行诊断历史且不得成为业务权威数据源
|
||||
|
||||
RocksDB 是唯一权威数据源
|
||||
|
||||
@@ -72,11 +75,22 @@ Bloom Filter 只能作为前置加速结构不得作为最终去重依据
|
||||
- `search` 只负责 Tantivy schema 文档转换索引和查询
|
||||
- `api` 只负责 HTTP 协议参数校验和响应转换
|
||||
- `config` 只负责读取校验和暴露配置
|
||||
- `diagnostics` 只负责采集聚合和查询可删除的运行指标历史
|
||||
- `telemetry` 只负责日志指标和运行观测
|
||||
- `shutdown` 只负责关闭信号和优雅退出协调
|
||||
|
||||
模块之间通过明确的数据结构和 trait 通信不得跨层直接访问内部实现
|
||||
|
||||
配置文件只是配置 DTO 的持久化适配器
|
||||
|
||||
用户配置 DTO 运行时解析结果和配置存储实现必须保持独立边界
|
||||
|
||||
配置更新必须先完整校验再通过同目录临时文件同步和原子替换保存
|
||||
|
||||
配置接口必须使用修订号阻止并发请求静默覆盖且不得假装未实际支持的在线热更新
|
||||
|
||||
领域层不得直接依赖 `dht-crawler` 的回调或传输 DTO
|
||||
|
||||
## 组合和文件边界
|
||||
|
||||
- trait 只用于存储搜索网络回调等真实替换边界 不创建只有一个调用方的抽象基类或通用 Service 层
|
||||
@@ -98,16 +112,17 @@ Bloom Filter 只能作为前置加速结构不得作为最终去重依据
|
||||
- Tantivy 写入使用批量提交并明确控制 IndexWriter 内存预算
|
||||
- 日志不得输出完整 Metadata 或大文件列表
|
||||
- 长期运行的集合必须有容量上限过期规则或磁盘持久化方案
|
||||
- 诊断历史必须使用独立 SQLite 数据库并通过采样降级和保留策略限制增长
|
||||
|
||||
## 开发规则
|
||||
|
||||
新业务代码写入 `dht-search`
|
||||
新业务代码写入 `src/search`
|
||||
|
||||
项目阶段任务完成状态和验收标准统一维护在根目录 `TODOS.md`
|
||||
|
||||
需求实现或技术决策发生变化时必须同步更新 `TODOS.md`
|
||||
|
||||
`dht-crawler` 只接受可复用的 DHT 基础能力不得包含数据库搜索接口或部署逻辑
|
||||
`src/crawler` 只接受可复用的 DHT 基础能力不得包含数据库搜索接口或部署逻辑
|
||||
|
||||
每个 Rust 文件顶部必须使用中文行注释描述该文件的功能边界且注释行尾不添加标点
|
||||
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
本文档保存可重复的规模基准条件目标和已验证结果
|
||||
|
||||
完整运行方式和参数说明见 [`dht-search/README.md`](dht-search/README.md)
|
||||
完整运行方式和参数说明见 [`src/search/README.md`](src/search/README.md)
|
||||
|
||||
## 基准条件
|
||||
|
||||
|
||||
Generated
+74
@@ -659,9 +659,11 @@ dependencies = [
|
||||
"dunce",
|
||||
"fs2",
|
||||
"hex",
|
||||
"libc",
|
||||
"regex",
|
||||
"rmp-serde",
|
||||
"rocksdb",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tantivy",
|
||||
@@ -787,6 +789,18 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "fastdivide"
|
||||
version = "0.4.2"
|
||||
@@ -1051,6 +1065,18 @@ name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
@@ -1438,6 +1464,17 @@ dependencies = [
|
||||
"lz4-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.38.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.29"
|
||||
@@ -2155,6 +2192,31 @@ dependencies = [
|
||||
"librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsqlite-vfs"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.40.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
"sqlite-wasm-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-stemmers"
|
||||
version = "1.2.0"
|
||||
@@ -2454,6 +2516,18 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-wasm-rs"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"js-sys",
|
||||
"rsqlite-vfs",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
members = ["dht-crawler", "dht-search"]
|
||||
members = ["src/crawler", "src/search"]
|
||||
exclude = ["opencodes"]
|
||||
resolver = "3"
|
||||
|
||||
|
||||
@@ -5,15 +5,15 @@
|
||||
## 目录
|
||||
|
||||
```text
|
||||
dht-search/ 最终运行的采集存储搜索和接口应用
|
||||
dht-crawler/ 可独立复用的 DHT 协议与 Metadata 获取基础库
|
||||
web/ 基于 Vue 和 shadcn-vue 的本地搜索界面
|
||||
src/search/ 最终运行的采集存储搜索和接口应用
|
||||
src/crawler/ 可独立复用的 DHT 协议与 Metadata 获取基础库
|
||||
src/web/ 基于 Vue 和 shadcn-vue 的本地搜索界面
|
||||
opencodes/ 不参与构建的参考项目
|
||||
```
|
||||
|
||||
当前已完成 DHT 采集持久化全文搜索内容聚合可用性验证 HTTP API 和本地 Web 搜索界面
|
||||
当前已完成 DHT 采集持久化全文搜索内容聚合可用性验证 HTTP API 本地 Web 搜索界面 有界 SQLite 运行诊断历史和配置管理
|
||||
|
||||
基础库的使用方式和指标说明见 [`dht-crawler/README.md`](dht-crawler/README.md)
|
||||
基础库的使用方式和指标说明见 [`src/crawler/README.md`](src/crawler/README.md)
|
||||
|
||||
当前实施阶段和后续计划见 [`TODOS.md`](TODOS.md)
|
||||
|
||||
@@ -25,10 +25,12 @@ Rust 测试分层和默认验证命令见 [`TESTING.md`](TESTING.md)
|
||||
|
||||
无效文件隐藏规则见 [`content-filters.toml`](content-filters.toml)
|
||||
|
||||
应用构建运行和 API 文档见 [`dht-search/README.md`](dht-search/README.md)
|
||||
应用构建运行和 API 文档见 [`src/search/README.md`](src/search/README.md)
|
||||
|
||||
Web 开发和构建方式见 [`web/README.md`](web/README.md)
|
||||
Web 开发和构建方式见 [`src/web/README.md`](src/web/README.md)
|
||||
|
||||
开发环境可以直接运行 `scripts\run.bat` 在当前窗口同时启动 Rust 后端和 Web 前端 按一次 `Ctrl+C` 即可统一停止
|
||||
|
||||
启动脚本使用本地 `dht-search.toml` 文件 文件不存在时会从 `dht-search.example.toml` 创建 因此 Web 配置页不会修改版本库中的模板
|
||||
|
||||
任一服务异常退出时启动脚本会清理 Cargo Bun 及其子进程树 避免遗留 Vite 或后端进程
|
||||
|
||||
+3
-1
@@ -18,11 +18,13 @@
|
||||
|
||||
RocksDB adapter 测试需要验证原子批处理私有键空间租约和损坏状态 因此保留在 `storage::rocks` 内部
|
||||
|
||||
SQLite 诊断组件测试需要验证 WAL 持久化分钟合并保留清理和 writer 关闭 因此保留在 `diagnostics` 模块内部
|
||||
|
||||
## 集成测试
|
||||
|
||||
只使用 crate 公开 API 的跨层契约放在 crate 根目录 `tests/`
|
||||
|
||||
`dht-search/tests/storage_search_flow.rs` 从外部组合领域模型 RocksDB repository 和 Tantivy search 验证写入索引查询关闭重开和恢复
|
||||
`src/search/tests/storage_search_flow.rs` 从外部组合领域模型 RocksDB repository 和 Tantivy search 验证写入索引查询关闭重开和恢复
|
||||
|
||||
集成测试不得依赖 `pub(crate)` 或为测试扩大生产 API 可见性
|
||||
|
||||
|
||||
@@ -16,11 +16,14 @@
|
||||
|
||||
## 当前技术方向
|
||||
|
||||
- `dht-crawler` 负责可复用的 DHT 协议节点发现 Peer 查找和 Metadata 下载
|
||||
- `dht-search` 负责持久化去重索引搜索接口配置和运行生命周期
|
||||
- `src/crawler` 负责可复用的 DHT 协议节点发现 Peer 查找和 Metadata 下载
|
||||
- `src/search` 负责持久化去重索引搜索接口配置和运行生命周期
|
||||
- `src/web` 负责最终用户搜索诊断和配置管理界面
|
||||
- RocksDB 保存权威数据去重信息和任务状态
|
||||
- Tantivy 保存可以从 RocksDB 重建的搜索索引
|
||||
- Axum 提供搜索详情统计和健康检查接口
|
||||
- 用户配置使用强类型 DTO 表达 TOML 只是当前持久化适配器
|
||||
- SQLite 保存有明确保留上限且可安全删除的运行诊断历史
|
||||
- 所有长期任务通过有界队列和背压控制资源占用
|
||||
|
||||
如果实际运行证明 RocksDB 的构建部署或资源成本不合适可以重新评估 redb SQLite 或其他存储方案
|
||||
@@ -33,7 +36,7 @@
|
||||
|
||||
### 任务
|
||||
|
||||
- [x] 将 workspace 扁平化为 `dht-crawler` 和 `dht-search`
|
||||
- [x] 将 crawler search 和 web 源码统一收纳到根目录 `src`
|
||||
- [x] 使用当前 Git 配置统一作者仓库许可证和 edition 元数据
|
||||
- [x] 编写 `AGENTS.md` 记录架构边界和开发约定
|
||||
- [x] 将最终应用与可复用 DHT 基础库分离
|
||||
@@ -183,7 +186,7 @@
|
||||
- [x] 修复一键启动脚本只停止父进程导致 Vite 子进程残留的问题
|
||||
- [x] 定义统一错误响应
|
||||
- [x] 限制查询长度分页大小和最大 offset
|
||||
- [ ] 增加请求延迟错误率和并发指标
|
||||
- [x] 增加请求延迟错误率和并发指标
|
||||
- [x] 增加搜索详情字段和按需验证入队 API 端到端测试
|
||||
- [x] 增加搜索过滤折叠精确哈希和变体接口测试
|
||||
|
||||
@@ -333,7 +336,18 @@
|
||||
- [x] 将规模基准拆分为参数数据集工作负载采样报告和编排模块
|
||||
- [x] 明确单元组件集成端到端和性能测试层级并增加公开 API 集成测试
|
||||
- [x] 实现可回滚的无效文件过滤并重建有效内容聚合和搜索索引
|
||||
- [x] 将 crawler search 和 web 统一迁移到根目录 `src` 并修复构建脚本文档路径
|
||||
- [x] 使用领域 Metadata DTO 切断领域层对 DHT 传输 DTO 的直接依赖
|
||||
- [x] 将配置拆分为可序列化 DTO TOML 读取适配器和运行时解析结果
|
||||
- [x] 使用独立 SQLite 建立有界运行诊断历史存储
|
||||
- [x] 采集进程 RocksDB Tantivy DHT 队列和磁盘资源快照
|
||||
- [x] 提供当前诊断快照和原始或分钟历史查询接口
|
||||
- [x] 将 HTTP 请求并发客户端错误服务端错误和延迟分布写入诊断历史
|
||||
- [x] 增加配置查询完整校验原子保存并发修订和统一重启提示
|
||||
- [x] 增加 Web 诊断页和配置管理页
|
||||
|
||||
完成二十四小时持续运行并继续观察私有内存 Metadata 成功率候选队列深度和每条成功 Metadata 的网络成本
|
||||
|
||||
随后补充 RocksDB 和 Tantivy 内部资源指标并根据实测继续调优
|
||||
RocksDB Tantivy HTTP 和进程资源指标已经接入诊断历史 后续根据长期实测继续调优
|
||||
|
||||
下一轮结构优化优先拆分 crawler 中的运行统计调度器和抓取引擎以及 search 中的 RocksDB 适配器 不为拆分而新增 crate
|
||||
|
||||
@@ -20,6 +20,14 @@ interval_secs = 21600
|
||||
retain_checkpoints = 3
|
||||
create_on_start = true
|
||||
|
||||
[diagnostics]
|
||||
enabled = true
|
||||
database = "data/diagnostics.sqlite3"
|
||||
sample_interval_secs = 10
|
||||
raw_retention_hours = 24
|
||||
minute_retention_days = 30
|
||||
queue_capacity = 128
|
||||
|
||||
[logging]
|
||||
directory = "data/logs"
|
||||
file_enabled = true
|
||||
@@ -58,7 +66,7 @@ new_destinations_per_minute = 12000
|
||||
|
||||
[http]
|
||||
listen = "127.0.0.1:8080"
|
||||
web_dir = "web/dist"
|
||||
web_dir = "src/web/dist"
|
||||
|
||||
[verification]
|
||||
enabled = true
|
||||
|
||||
@@ -10,4 +10,4 @@ max_path_depth = 4
|
||||
|
||||
[http]
|
||||
listen = "127.0.0.1:8080"
|
||||
web_dir = "web/dist"
|
||||
web_dir = "src/web/dist"
|
||||
|
||||
@@ -1,637 +0,0 @@
|
||||
// 负责加载校验和提供应用配置但不执行任何业务逻辑
|
||||
|
||||
use std::{fs, net::SocketAddr, path::PathBuf};
|
||||
|
||||
use clap::Parser;
|
||||
use dht_crawler::{
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, MetadataOptions, NetMode, PeerLookupOptions,
|
||||
PoolOptions, RateLimitOptions, SampleInfohashesOptions, SchedulerOptions, TargetOptions,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::AppError;
|
||||
use dht_search::domain::{ContentFilter, ContentFilterConfig, MetadataLimits};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "dht-search", version, about = "DHT 元数据采集和搜索服务")]
|
||||
pub(crate) struct Cli {
|
||||
#[arg(long, default_value = "dht-search.toml")]
|
||||
config: PathBuf,
|
||||
#[arg(long)]
|
||||
data_dir: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
run_duration_secs: Option<u64>,
|
||||
#[arg(long)]
|
||||
restore_checkpoint: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct StartupConfig {
|
||||
pub(crate) app: AppConfig,
|
||||
pub(crate) restore_checkpoint: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct AppConfig {
|
||||
pub(crate) data_dir: PathBuf,
|
||||
pub(crate) content_filter_file: PathBuf,
|
||||
pub(crate) persistence_queue_capacity: usize,
|
||||
pub(crate) stats_interval_secs: u64,
|
||||
pub(crate) run_duration_secs: Option<u64>,
|
||||
pub(crate) index_batch_size: usize,
|
||||
pub(crate) index_interval_millis: u64,
|
||||
pub(crate) metadata_limits: MetadataLimitsConfig,
|
||||
pub(crate) dht: DhtConfig,
|
||||
pub(crate) disk_guard: DiskGuardConfig,
|
||||
pub(crate) backup: BackupConfig,
|
||||
pub(crate) logging: LoggingConfig,
|
||||
pub(crate) http: HttpConfig,
|
||||
pub(crate) verification: VerificationConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct DhtConfig {
|
||||
pub(crate) port: u16,
|
||||
pub(crate) netmode: NetworkMode,
|
||||
pub(crate) hash_queue_capacity: usize,
|
||||
pub(crate) max_outbound_queries_per_second: u32,
|
||||
pub(crate) outbound_query_burst: u32,
|
||||
pub(crate) metadata_timeout_secs: u64,
|
||||
pub(crate) metadata_queue_capacity: usize,
|
||||
pub(crate) metadata_workers: usize,
|
||||
pub(crate) metadata_connects_per_second: u32,
|
||||
pub(crate) sample_queries_per_second: u32,
|
||||
pub(crate) sample_max_in_flight: usize,
|
||||
pub(crate) sample_new_node_percent: u8,
|
||||
pub(crate) sample_candidate_queue_capacity: usize,
|
||||
pub(crate) sample_fallback_to_iterative: bool,
|
||||
pub(crate) peer_lookups_per_second: u32,
|
||||
pub(crate) peer_lookup_max_active: usize,
|
||||
pub(crate) find_node_queries_per_second: u32,
|
||||
pub(crate) find_node_max_in_flight: usize,
|
||||
pub(crate) new_destinations_per_minute: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct HttpConfig {
|
||||
pub(crate) listen: SocketAddr,
|
||||
pub(crate) web_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct DiskGuardConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) check_interval_secs: u64,
|
||||
pub(crate) minimum_free_bytes: u64,
|
||||
pub(crate) resume_free_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct LoggingConfig {
|
||||
pub(crate) directory: PathBuf,
|
||||
pub(crate) file_enabled: bool,
|
||||
pub(crate) console_enabled: bool,
|
||||
pub(crate) rotation: LogRotation,
|
||||
pub(crate) retain_files: usize,
|
||||
pub(crate) file_prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) directory: PathBuf,
|
||||
pub(crate) interval_secs: u64,
|
||||
pub(crate) retain_checkpoints: usize,
|
||||
pub(crate) create_on_start: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum LogRotation {
|
||||
Minutely,
|
||||
Hourly,
|
||||
#[default]
|
||||
Daily,
|
||||
Never,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct VerificationConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) queue_capacity: usize,
|
||||
pub(crate) max_active: usize,
|
||||
pub(crate) max_peer_attempts: usize,
|
||||
pub(crate) lease_secs: u64,
|
||||
pub(crate) poll_interval_millis: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct MetadataLimitsConfig {
|
||||
pub(crate) max_metadata_bytes: usize,
|
||||
pub(crate) max_files: usize,
|
||||
pub(crate) max_name_bytes: usize,
|
||||
pub(crate) max_path_bytes: usize,
|
||||
pub(crate) max_path_depth: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum NetworkMode {
|
||||
#[default]
|
||||
Ipv4Only,
|
||||
Ipv6Only,
|
||||
DualStack,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub(crate) fn load(self) -> Result<StartupConfig, AppError> {
|
||||
let config_path = absolute_path(&self.config)?;
|
||||
let mut config = if config_path.exists() {
|
||||
let contents = fs::read_to_string(&config_path)?;
|
||||
toml::from_str::<AppConfig>(&contents)?
|
||||
} else {
|
||||
AppConfig::default()
|
||||
};
|
||||
|
||||
if let Some(data_dir) = self.data_dir {
|
||||
config.data_dir = data_dir;
|
||||
}
|
||||
if self.run_duration_secs.is_some() {
|
||||
config.run_duration_secs = self.run_duration_secs;
|
||||
}
|
||||
let base = config_path
|
||||
.parent()
|
||||
.ok_or_else(|| AppError::Config("配置文件没有父目录".to_owned()))?;
|
||||
if config.data_dir.is_relative() {
|
||||
config.data_dir = base.join(&config.data_dir);
|
||||
}
|
||||
config.data_dir = normalize_absolute(config.data_dir)?;
|
||||
if config.content_filter_file.is_relative() {
|
||||
config.content_filter_file = base.join(&config.content_filter_file);
|
||||
}
|
||||
config.content_filter_file = normalize_absolute(config.content_filter_file)?;
|
||||
if config.logging.directory.is_relative() {
|
||||
config.logging.directory = base.join(&config.logging.directory);
|
||||
}
|
||||
config.logging.directory = normalize_absolute(config.logging.directory)?;
|
||||
if config.backup.directory.is_relative() {
|
||||
config.backup.directory = base.join(&config.backup.directory);
|
||||
}
|
||||
config.backup.directory = normalize_absolute(config.backup.directory)?;
|
||||
if config.http.web_dir.is_relative() {
|
||||
config.http.web_dir = base.join(&config.http.web_dir);
|
||||
}
|
||||
config.http.web_dir = normalize_absolute(config.http.web_dir)?;
|
||||
config.validate()?;
|
||||
let restore_checkpoint = self
|
||||
.restore_checkpoint
|
||||
.map(normalize_absolute)
|
||||
.transpose()?;
|
||||
Ok(StartupConfig {
|
||||
app: config,
|
||||
restore_checkpoint,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
pub(crate) fn content_filter(&self) -> Result<ContentFilter, AppError> {
|
||||
let contents = fs::read_to_string(&self.content_filter_file)?;
|
||||
let config = toml::from_str::<ContentFilterConfig>(&contents)?;
|
||||
ContentFilter::compile(config).map_err(AppError::from)
|
||||
}
|
||||
|
||||
pub(crate) fn dht_options(&self) -> DHTOptions {
|
||||
let defaults = DHTOptions::default();
|
||||
DHTOptions {
|
||||
port: self.dht.port,
|
||||
netmode: self.dht.netmode.into(),
|
||||
hash_queue_capacity: self.dht.hash_queue_capacity,
|
||||
max_outbound_queries_per_second: self.dht.max_outbound_queries_per_second,
|
||||
outbound_query_burst: self.dht.outbound_query_burst,
|
||||
metadata: MetadataOptions {
|
||||
timeout_secs: self.dht.metadata_timeout_secs,
|
||||
max_queue_size: self.dht.metadata_queue_capacity,
|
||||
max_worker_count: self.dht.metadata_workers,
|
||||
max_connects_per_second: self.dht.metadata_connects_per_second,
|
||||
max_metadata_size_bytes: self.metadata_limits.max_metadata_bytes,
|
||||
..defaults.metadata
|
||||
},
|
||||
peer_lookup: PeerLookupOptions {
|
||||
max_lookups_per_second: self.dht.peer_lookups_per_second,
|
||||
burst: self.dht.peer_lookups_per_second.max(1),
|
||||
max_active_lookups: self.dht.peer_lookup_max_active,
|
||||
},
|
||||
sample_infohashes: SampleInfohashesOptions {
|
||||
max_queries_per_second: self.dht.sample_queries_per_second,
|
||||
burst: self.dht.sample_queries_per_second.max(1),
|
||||
max_in_flight: self.dht.sample_max_in_flight,
|
||||
new_node_sample_percent: self.dht.sample_new_node_percent,
|
||||
candidate_queue_capacity: self.dht.sample_candidate_queue_capacity,
|
||||
fallback_to_iterative: self.dht.sample_fallback_to_iterative,
|
||||
..defaults.sample_infohashes
|
||||
},
|
||||
crawl: CrawlOptions {
|
||||
pool: PoolOptions {
|
||||
..defaults.crawl.pool
|
||||
},
|
||||
rate_limit: RateLimitOptions {
|
||||
max_find_node_rate_per_sec: self.dht.find_node_queries_per_second,
|
||||
burst: self.dht.outbound_query_burst,
|
||||
max_in_flight: self.dht.find_node_max_in_flight,
|
||||
max_new_destinations_per_minute: self.dht.new_destinations_per_minute,
|
||||
..defaults.crawl.rate_limit
|
||||
},
|
||||
bootstrap: BootstrapOptions {
|
||||
..defaults.crawl.bootstrap
|
||||
},
|
||||
target: TargetOptions {
|
||||
..defaults.crawl.target
|
||||
},
|
||||
scheduler: SchedulerOptions {
|
||||
..defaults.crawl.scheduler
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metadata_limits(&self) -> MetadataLimits {
|
||||
MetadataLimits {
|
||||
max_files: self.metadata_limits.max_files,
|
||||
max_name_bytes: self.metadata_limits.max_name_bytes,
|
||||
max_path_bytes: self.metadata_limits.max_path_bytes,
|
||||
max_path_depth: self.metadata_limits.max_path_depth,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<(), AppError> {
|
||||
if self.persistence_queue_capacity == 0 {
|
||||
return Err(AppError::Config(
|
||||
"persistence_queue_capacity 必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.stats_interval_secs == 0 {
|
||||
return Err(AppError::Config(
|
||||
"stats_interval_secs 必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.run_duration_secs == Some(0) {
|
||||
return Err(AppError::Config(
|
||||
"run_duration_secs 必须大于零或不设置".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.index_batch_size == 0 || self.index_interval_millis == 0 {
|
||||
return Err(AppError::Config(
|
||||
"索引批量大小和执行间隔必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.metadata_limits.max_metadata_bytes == 0
|
||||
|| self.metadata_limits.max_files == 0
|
||||
|| self.metadata_limits.max_name_bytes == 0
|
||||
|| self.metadata_limits.max_path_bytes == 0
|
||||
|| self.metadata_limits.max_path_depth == 0
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"Metadata 大小文件数名称路径和目录层级上限必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.dht.metadata_workers == 0 || self.dht.metadata_queue_capacity == 0 {
|
||||
return Err(AppError::Config(
|
||||
"Metadata worker 和队列容量必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.verification.queue_capacity == 0
|
||||
|| self.verification.max_active == 0
|
||||
|| self.verification.max_peer_attempts == 0
|
||||
|| self.verification.lease_secs == 0
|
||||
|| self.verification.poll_interval_millis == 0
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"验证队列容量并发尝试数租约和轮询间隔必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.disk_guard.check_interval_secs == 0
|
||||
|| self.disk_guard.minimum_free_bytes == 0
|
||||
|| self.disk_guard.resume_free_bytes <= self.disk_guard.minimum_free_bytes
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"磁盘检查间隔必须大于零且恢复阈值必须大于保护阈值".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.backup.interval_secs == 0 || self.backup.retain_checkpoints == 0 {
|
||||
return Err(AppError::Config(
|
||||
"备份间隔和检查点保留数量必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
let database_path = self.data_dir.join("rocksdb");
|
||||
if self.backup.directory == database_path
|
||||
|| self.backup.directory.starts_with(&database_path)
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"检查点目录不能位于 RocksDB 数据库目录内部".to_owned(),
|
||||
));
|
||||
}
|
||||
if !self.logging.file_enabled && !self.logging.console_enabled {
|
||||
return Err(AppError::Config(
|
||||
"文件日志和终端日志不能同时关闭".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.logging.file_enabled && self.logging.retain_files == 0 {
|
||||
return Err(AppError::Config("日志保留文件数量必须大于零".to_owned()));
|
||||
}
|
||||
if self.logging.file_prefix.trim().is_empty()
|
||||
|| self
|
||||
.logging
|
||||
.file_prefix
|
||||
.chars()
|
||||
.any(|character| character.is_control() || matches!(character, '/' | '\\'))
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"日志文件前缀不能为空且不能包含路径分隔符或控制字符".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.dht.max_outbound_queries_per_second == 0
|
||||
|| self.dht.outbound_query_burst == 0
|
||||
|| self.dht.metadata_connects_per_second == 0
|
||||
|| self.dht.sample_max_in_flight == 0
|
||||
|| self.dht.sample_candidate_queue_capacity == 0
|
||||
|| self.dht.peer_lookup_max_active == 0
|
||||
|| self.dht.find_node_max_in_flight == 0
|
||||
{
|
||||
return Err(AppError::Config("网络速率和并发上限必须大于零".to_owned()));
|
||||
}
|
||||
if self.dht.sample_new_node_percent > 100 {
|
||||
return Err(AppError::Config(
|
||||
"sample_new_node_percent 必须在 0 到 100 之间".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AppConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
data_dir: PathBuf::from("data"),
|
||||
content_filter_file: PathBuf::from("content-filters.toml"),
|
||||
persistence_queue_capacity: 8_192,
|
||||
stats_interval_secs: 10,
|
||||
run_duration_secs: None,
|
||||
index_batch_size: 1_024,
|
||||
index_interval_millis: 5_000,
|
||||
metadata_limits: MetadataLimitsConfig::default(),
|
||||
dht: DhtConfig::default(),
|
||||
disk_guard: DiskGuardConfig::default(),
|
||||
backup: BackupConfig::default(),
|
||||
logging: LoggingConfig::default(),
|
||||
http: HttpConfig::default(),
|
||||
verification: VerificationConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MetadataLimitsConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_metadata_bytes: 10 * 1024 * 1024,
|
||||
max_files: 20_000,
|
||||
max_name_bytes: 1_024,
|
||||
max_path_bytes: 4_096,
|
||||
max_path_depth: 64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DhtConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
port: 12_313,
|
||||
netmode: NetworkMode::Ipv4Only,
|
||||
hash_queue_capacity: 20_000,
|
||||
max_outbound_queries_per_second: 1_000,
|
||||
outbound_query_burst: 200,
|
||||
metadata_timeout_secs: 6,
|
||||
metadata_queue_capacity: 20_000,
|
||||
metadata_workers: 400,
|
||||
metadata_connects_per_second: 400,
|
||||
sample_queries_per_second: 60,
|
||||
sample_max_in_flight: 100,
|
||||
sample_new_node_percent: 50,
|
||||
sample_candidate_queue_capacity: 8_192,
|
||||
sample_fallback_to_iterative: false,
|
||||
peer_lookups_per_second: 200,
|
||||
peer_lookup_max_active: 200,
|
||||
find_node_queries_per_second: 10,
|
||||
find_node_max_in_flight: 100,
|
||||
new_destinations_per_minute: 12_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HttpConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
listen: SocketAddr::from(([127, 0, 0, 1], 8080)),
|
||||
web_dir: PathBuf::from("web/dist"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiskGuardConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
check_interval_secs: 10,
|
||||
minimum_free_bytes: 5 * 1024 * 1024 * 1024,
|
||||
resume_free_bytes: 6 * 1024 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
directory: PathBuf::from("data/logs"),
|
||||
file_enabled: true,
|
||||
console_enabled: false,
|
||||
rotation: LogRotation::Daily,
|
||||
retain_files: 7,
|
||||
file_prefix: "dht-search".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BackupConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
directory: PathBuf::from("data/backups"),
|
||||
interval_secs: 6 * 60 * 60,
|
||||
retain_checkpoints: 3,
|
||||
create_on_start: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for VerificationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
queue_capacity: 10_000,
|
||||
max_active: 8,
|
||||
max_peer_attempts: 3,
|
||||
lease_secs: 60,
|
||||
poll_interval_millis: 250,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NetworkMode> for NetMode {
|
||||
fn from(value: NetworkMode) -> Self {
|
||||
match value {
|
||||
NetworkMode::Ipv4Only => Self::Ipv4Only,
|
||||
NetworkMode::Ipv6Only => Self::Ipv6Only,
|
||||
NetworkMode::DualStack => Self::DualStack,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn absolute_path(path: &PathBuf) -> Result<PathBuf, AppError> {
|
||||
if path.is_absolute() {
|
||||
Ok(path.clone())
|
||||
} else {
|
||||
Ok(std::env::current_dir()?.join(path))
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_absolute(path: PathBuf) -> Result<PathBuf, AppError> {
|
||||
if path.is_absolute() {
|
||||
Ok(path)
|
||||
} else {
|
||||
absolute_path(&path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn defaults_produce_valid_dht_options() {
|
||||
let config = AppConfig::default();
|
||||
config.validate().unwrap();
|
||||
let options = config.dht_options();
|
||||
assert_eq!(options.port, 12_313);
|
||||
assert_eq!(options.metadata.max_worker_count, 400);
|
||||
assert_eq!(options.metadata.max_connects_per_second, 400);
|
||||
assert_eq!(options.metadata.max_metadata_size_bytes, 10 * 1024 * 1024);
|
||||
assert_eq!(options.max_outbound_queries_per_second, 1_000);
|
||||
assert_eq!(options.sample_infohashes.max_queries_per_second, 60);
|
||||
assert_eq!(options.sample_infohashes.max_in_flight, 100);
|
||||
assert_eq!(options.sample_infohashes.new_node_sample_percent, 50);
|
||||
assert_eq!(options.sample_infohashes.candidate_queue_capacity, 8_192);
|
||||
assert!(!options.sample_infohashes.fallback_to_iterative);
|
||||
assert_eq!(options.peer_lookup.max_lookups_per_second, 200);
|
||||
assert_eq!(options.peer_lookup.max_active_lookups, 200);
|
||||
assert_eq!(options.crawl.rate_limit.max_find_node_rate_per_sec, 10);
|
||||
assert_eq!(
|
||||
options.crawl.rate_limit.max_new_destinations_per_minute,
|
||||
12_000
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_data_directory_is_resolved_from_config_file() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let config_path = directory.path().join("service.toml");
|
||||
fs::write(&config_path, "data_dir = 'state'").unwrap();
|
||||
let startup = Cli {
|
||||
config: config_path,
|
||||
data_dir: None,
|
||||
run_duration_secs: None,
|
||||
restore_checkpoint: None,
|
||||
}
|
||||
.load()
|
||||
.unwrap();
|
||||
let config = startup.app;
|
||||
assert_eq!(config.data_dir, directory.path().join("state"));
|
||||
assert_eq!(
|
||||
config.content_filter_file,
|
||||
directory.path().join("content-filters.toml")
|
||||
);
|
||||
assert_eq!(config.logging.directory, directory.path().join("data/logs"));
|
||||
assert_eq!(
|
||||
config.backup.directory,
|
||||
directory.path().join("data/backups")
|
||||
);
|
||||
assert_eq!(config.http.web_dir, directory.path().join("web/dist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_config_field_is_rejected() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let config_path = directory.path().join("service.toml");
|
||||
fs::write(&config_path, "unknown = true").unwrap();
|
||||
let error = Cli {
|
||||
config: config_path,
|
||||
data_dir: None,
|
||||
run_duration_secs: None,
|
||||
restore_checkpoint: None,
|
||||
}
|
||||
.load()
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, AppError::Toml(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_metadata_limit_is_rejected() {
|
||||
let mut config = AppConfig::default();
|
||||
config.metadata_limits.max_files = 0;
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_new_node_sample_percent_is_rejected() {
|
||||
let mut config = AppConfig::default();
|
||||
config.dht.sample_new_node_percent = 101;
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_resume_threshold_must_exceed_minimum() {
|
||||
let mut config = AppConfig::default();
|
||||
config.disk_guard.resume_free_bytes = config.disk_guard.minimum_free_bytes;
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_output() {
|
||||
let mut config = AppConfig::default();
|
||||
config.logging.file_enabled = false;
|
||||
config.logging.console_enabled = false;
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_prefix_cannot_escape_the_log_directory() {
|
||||
let mut config = AppConfig::default();
|
||||
config.logging.file_prefix = "../service".into();
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_directory_cannot_be_inside_rocksdb() {
|
||||
let mut config = AppConfig::default();
|
||||
config.backup.directory = config.data_dir.join("rocksdb/checkpoints");
|
||||
assert!(matches!(config.validate(), Err(AppError::Config(_))));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
// 负责导出可测试可组合的领域模型和持久化能力
|
||||
|
||||
pub mod domain;
|
||||
pub mod search;
|
||||
pub mod storage;
|
||||
+12
-2
@@ -2,7 +2,7 @@
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..") do set "PROJECT_ROOT=%%~fI"
|
||||
set "WEB_DIR=%PROJECT_ROOT%\web"
|
||||
set "WEB_DIR=%PROJECT_ROOT%\src\web"
|
||||
set "LIBCLANG_PATH=%PROJECT_ROOT%\.tools\libclang\clang\native"
|
||||
|
||||
where cargo >nul 2>nul
|
||||
@@ -29,6 +29,16 @@ if not exist "%LIBCLANG_PATH%\libclang.dll" (
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if not exist "%PROJECT_ROOT%\dht-search.toml" (
|
||||
echo [INFO] Creating local configuration from template
|
||||
copy /Y "%PROJECT_ROOT%\dht-search.example.toml" "%PROJECT_ROOT%\dht-search.toml" >nul
|
||||
if errorlevel 1 (
|
||||
echo [ERROR] Failed to create dht-search.toml
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
)
|
||||
|
||||
if not exist "%WEB_DIR%\node_modules" (
|
||||
echo [INFO] Installing Web dependencies
|
||||
pushd "%WEB_DIR%"
|
||||
@@ -60,7 +70,7 @@ powershell -NoProfile -ExecutionPolicy Bypass -Command ^
|
||||
" }" ^
|
||||
"}" ^
|
||||
"try {" ^
|
||||
" $backend = Start-Process -FilePath 'cargo' -ArgumentList @('run','-p','dht-search','--','--config','dht-search.example.toml') -WorkingDirectory $env:PROJECT_ROOT -NoNewWindow -PassThru;" ^
|
||||
" $backend = Start-Process -FilePath 'cargo' -ArgumentList @('run','-p','dht-search','--bin','dht-search','--','--config','dht-search.toml') -WorkingDirectory $env:PROJECT_ROOT -NoNewWindow -PassThru;" ^
|
||||
" $web = Start-Process -FilePath 'bun' -ArgumentList @('--bun','run','dev','--','--host','127.0.0.1') -WorkingDirectory $env:WEB_DIR -NoNewWindow -PassThru;" ^
|
||||
" while (-not $backend.HasExited -and -not $web.HasExited) { Start-Sleep -Milliseconds 250 }" ^
|
||||
"} finally {" ^
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
[](https://crates.io/crates/dht-crawler)
|
||||
[](https://docs.rs/dht-crawler)
|
||||
[](../LICENSE)
|
||||
[](../../LICENSE)
|
||||
|
||||
基于 Rust 和 Tokio 的 BitTorrent DHT 爬虫库。它参与 BEP-5 DHT 网络，通过 BEP-51
|
||||
`sample_infohashes` 主动发现 InfoHash，也接收 `announce_peer`，并通过 BEP-9
|
||||
@@ -38,12 +38,12 @@ pub use runtime_stats::{
|
||||
DhtObservabilitySnapshot, DhtRuntimeSnapshot, DhtRuntimeStats, FixedHistogramSnapshot,
|
||||
};
|
||||
pub use scheduler::{MetadataScheduler, MetadataSchedulerCallbacks, MetadataSchedulerLimits};
|
||||
pub use server::{DHTServer, HashDiscovered};
|
||||
pub use server::DHTServer;
|
||||
pub use types::{
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, DiscoverySource, FileInfo, MetadataFetchCompletion,
|
||||
MetadataFetchCompletionStatus, MetadataOptions, NetMode, NodeTuple, PeerLookupOptions,
|
||||
PeerLookupResult, PoolOptions, RateLimitOptions, SampleInfohashesOptions, SchedulerOptions,
|
||||
TargetOptions, TorrentInfo,
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, DiscoverySource, FileInfo, HashDiscovered,
|
||||
MetadataFetchCompletion, MetadataFetchCompletionStatus, MetadataOptions, NetMode, NodeTuple,
|
||||
PeerLookupOptions, PeerLookupResult, PoolOptions, RateLimitOptions, SampleInfohashesOptions,
|
||||
SchedulerOptions, TargetOptions, TorrentInfo,
|
||||
};
|
||||
|
||||
/// Common server, configuration and callback payload imports.
|
||||
@@ -55,7 +55,7 @@ pub mod prelude {
|
||||
};
|
||||
pub use crate::server::DHTServer;
|
||||
pub use crate::types::{
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, DiscoverySource, FileInfo,
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, DiscoverySource, FileInfo, HashDiscovered,
|
||||
MetadataFetchCompletion, MetadataFetchCompletionStatus, MetadataOptions, NetMode,
|
||||
NodeTuple, PeerLookupOptions, PeerLookupResult, PoolOptions, RateLimitOptions,
|
||||
SampleInfohashesOptions, SchedulerOptions, TargetOptions, TorrentInfo,
|
||||
@@ -6,8 +6,9 @@ use crate::node_id::TransactionId;
|
||||
use crate::protocol::DhtResponse;
|
||||
use crate::routing_snapshot::{RoutingSnapshot, xor_distance_cmp};
|
||||
use crate::runtime_stats::DhtRuntimeStats;
|
||||
use crate::server::HashDiscovered;
|
||||
use crate::types::{DiscoverySource, NetMode, NodeTuple, PeerLookupOptions, PeerLookupResult};
|
||||
use crate::types::{
|
||||
DiscoverySource, HashDiscovered, NetMode, NodeTuple, PeerLookupOptions, PeerLookupResult,
|
||||
};
|
||||
use ahash::{AHashMap, AHashSet};
|
||||
use arc_swap::ArcSwap;
|
||||
use bytes::BytesMut;
|
||||
@@ -5,9 +5,9 @@ use crate::peer_lookup::PeerLookupRequest;
|
||||
#[cfg(test)]
|
||||
use crate::runtime_stats::DhtRuntimeLimits;
|
||||
use crate::runtime_stats::DhtRuntimeStats;
|
||||
use crate::server::HashDiscovered;
|
||||
use crate::types::{
|
||||
DiscoverySource, MetadataFetchCompletion, MetadataFetchCompletionStatus, TorrentInfo,
|
||||
DiscoverySource, HashDiscovered, MetadataFetchCompletion, MetadataFetchCompletionStatus,
|
||||
TorrentInfo,
|
||||
};
|
||||
use arc_swap::ArcSwapOption;
|
||||
#[cfg(feature = "metrics")]
|
||||
@@ -27,7 +27,8 @@ use crate::scheduler::{
|
||||
TorrentAckCallback,
|
||||
};
|
||||
use crate::types::{
|
||||
DHTOptions, DiscoverySource, MetadataFetchCompletion, NetMode, NodeTuple, TorrentInfo,
|
||||
DHTOptions, DiscoverySource, HashDiscovered, MetadataFetchCompletion, NetMode, NodeTuple,
|
||||
TorrentInfo,
|
||||
};
|
||||
use crate::udp_buffer::UdpBufferPool;
|
||||
use crate::udp_ingress::{WorkerHandle, spawn_udp_listener};
|
||||
@@ -60,19 +61,6 @@ struct QueryResponse<'a> {
|
||||
target_id: Option<&'a [u8]>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
/// InfoHash and announcing Peer submitted to the Metadata scheduler.
|
||||
pub struct HashDiscovered {
|
||||
/// Lowercase hexadecimal InfoHash.
|
||||
pub info_hash: String,
|
||||
/// Peer endpoint derived from announce `port`/`implied_port`.
|
||||
pub peer_addr: SocketAddr,
|
||||
/// Network path that supplied this Peer candidate.
|
||||
pub source: DiscoverySource,
|
||||
/// Monotonic discovery time used for freshness and queue ordering.
|
||||
pub discovered_at: std::time::Instant,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
/// Cloneable BEP-5 server handle and primary crate entry point.
|
||||
pub struct DHTServer {
|
||||
@@ -174,6 +174,19 @@ pub enum DiscoverySource {
|
||||
ActiveLookup,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
/// InfoHash and announcing Peer submitted to the Metadata scheduler.
|
||||
pub struct HashDiscovered {
|
||||
/// Lowercase hexadecimal InfoHash.
|
||||
pub info_hash: String,
|
||||
/// Peer endpoint derived from announce `port`/`implied_port`.
|
||||
pub peer_addr: SocketAddr,
|
||||
/// Network path that supplied this Peer candidate.
|
||||
pub source: DiscoverySource,
|
||||
/// Monotonic discovery time used for freshness and queue ordering.
|
||||
pub discovered_at: std::time::Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
/// Terminal result of one bounded active `get_peers` lookup.
|
||||
pub struct PeerLookupResult {
|
||||
@@ -11,19 +11,20 @@ publish = false
|
||||
|
||||
[features]
|
||||
default = ["rocksdb-storage"]
|
||||
rocksdb-storage = ["dep:rocksdb"]
|
||||
rocksdb-storage = ["dep:rocksdb", "dep:rusqlite"]
|
||||
|
||||
[dependencies]
|
||||
axum = "0.8.9"
|
||||
blake3 = "1.8.5"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
dht-crawler = { path = "../dht-crawler", features = ["metrics"] }
|
||||
dht-crawler = { path = "../crawler", features = ["metrics"] }
|
||||
dunce = "1.0"
|
||||
hex = "0.4"
|
||||
fs2 = "0.4"
|
||||
rocksdb = { version = "0.24.0", default-features = false, features = ["bindgen-runtime", "lz4"], optional = true }
|
||||
rmp-serde = "1.3"
|
||||
regex = "1.12"
|
||||
rusqlite = { version = "0.40.2", features = ["bundled"], optional = true }
|
||||
serde.workspace = true
|
||||
serde_json = "1.0"
|
||||
tantivy = "0.26.1"
|
||||
@@ -42,7 +43,15 @@ tempfile = "3.27"
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61.2", features = ["Win32_System_ProcessStatus", "Win32_System_Threading"] }
|
||||
windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Diagnostics_ToolHelp", "Win32_System_ProcessStatus", "Win32_System_Threading"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[[bin]]
|
||||
name = "dht-search"
|
||||
path = "src/main.rs"
|
||||
required-features = ["rocksdb-storage"]
|
||||
|
||||
[[bin]]
|
||||
name = "dht-benchmark"
|
||||
@@ -36,7 +36,7 @@ cargo run -p dht-search -- --data-dir D:\data\dht-search --run-duration-secs 360
|
||||
|
||||
不设置 `run-duration-secs` 时服务持续运行直到收到 Ctrl+C SIGINT 或 SIGTERM
|
||||
|
||||
生产 Web 页面需要先在 `web` 目录执行 `bun run build` Axum 会从 `http.web_dir` 提供构建结果
|
||||
生产 Web 页面需要先在 `src/web` 目录执行 `bun run build` Axum 会从 `http.web_dir` 提供构建结果
|
||||
|
||||
### 当前运行模板网络配置
|
||||
|
||||
@@ -69,7 +69,7 @@ Metadata 下载和可用性握手共用 `metadata_connects_per_second` 预算不
|
||||
|---|---:|---|
|
||||
| `verification.enabled` | `true` | 是否启用按需可用性验证 |
|
||||
| `verification.queue_capacity` | `10000` | 持久化验证队列容量 |
|
||||
| `verification.max_active` | `2` | 同时验证的种子数量 |
|
||||
| `verification.max_active` | `8` | 同时验证的种子数量 |
|
||||
| `verification.max_peer_attempts` | `3` | 每个种子最多握手的 Peer 数量 |
|
||||
| `verification.lease_secs` | `60` | 异常退出后验证任务重新可领取的租约时间 |
|
||||
| `verification.poll_interval_millis` | `250` | 持久化队列轮询间隔 |
|
||||
@@ -146,6 +146,33 @@ target\release\dht-search.exe `
|
||||
|
||||
确认恢复数据无误后可以人工删除 `rocksdb.pre-restore-*` 释放空间 不要在服务运行时移动或删除这些目录
|
||||
|
||||
### 运行诊断历史
|
||||
|
||||
应用把可删除的运行资源采样写入独立的 `data/diagnostics.sqlite3` RocksDB 仍然是唯一业务权威数据源 删除诊断数据库不会影响种子数据搜索索引或恢复
|
||||
|
||||
| 配置项 | 默认值 | 作用 |
|
||||
|---|---:|---|
|
||||
| `diagnostics.enabled` | `true` | 是否采集并保存运行诊断历史 |
|
||||
| `diagnostics.database` | `data/diagnostics.sqlite3` | 独立 SQLite 数据库路径 |
|
||||
| `diagnostics.sample_interval_secs` | `10` | 实时资源采样间隔 |
|
||||
| `diagnostics.raw_retention_hours` | `24` | 原始采样保留时间 |
|
||||
| `diagnostics.minute_retention_days` | `30` | 每分钟快照保留时间 |
|
||||
| `diagnostics.queue_capacity` | `128` | SQLite writer 有界队列容量 |
|
||||
|
||||
SQLite 使用 WAL 和单独 writer 线程 原始采样超过 24 小时自动删除 同一分钟只保留最新快照且超过 30 天自动删除 写入失败不会停止采集和搜索核心服务
|
||||
|
||||
诊断快照包含进程内存 CPU 时间线程句柄 DHT 流量队列深度 RocksDB Block Cache MemTable Compaction 和 SST 状态 Tantivy Writer 预算和提交耗时以及 HTTP 并发错误分类平均 P95 和最大延迟
|
||||
|
||||
### 配置管理
|
||||
|
||||
配置文件只是强类型配置 DTO 的 TOML 持久化形式 服务通过 `GET /config` 返回当前文件配置和稳定修订号 通过 `PUT /config` 接受完整 DTO
|
||||
|
||||
更新前会执行与启动时相同的完整校验 保存时先在同目录写入并同步临时文件再原子替换目标文件 旧修订号返回 `409 Conflict` 防止多个页面互相覆盖
|
||||
|
||||
当前版本不在线修改正在运行的 DHT 存储索引和监听器 保存成功后返回 `restart_required = true` 重启服务后统一生效 命令行覆盖字段也会单独返回并继续优先于文件配置
|
||||
|
||||
通过 Web 保存会按 DTO 重新生成 TOML 原有手写注释不会保留 管理接口默认随 HTTP 服务提供 因此生产部署不应把 `/config` 暴露到不受信任的公网入口
|
||||
|
||||
### Metadata 安全限制
|
||||
|
||||
应用会在 Metadata 下载和进入 RocksDB 前执行两层资源与结构校验
|
||||
@@ -225,6 +252,11 @@ LimitNOFILE=65536
|
||||
GET /health
|
||||
GET /ready
|
||||
GET /stats
|
||||
GET /diagnostics/current
|
||||
GET /diagnostics/history?range_secs=3600&resolution=raw
|
||||
GET /diagnostics/history?range_secs=86400&resolution=minute
|
||||
GET /config
|
||||
PUT /config
|
||||
GET /search?q=ubuntu&offset=0&limit=20
|
||||
GET /search?q=%2A.iso
|
||||
GET /search?q=&min_size=1048576&max_size=10737418240&extension=mkv
|
||||
@@ -254,6 +286,8 @@ GET /torrents/{infohash}?file_offset=0&file_limit=100
|
||||
|
||||
详情文件列表默认返回 100 条且单次最多 200 条 使用 `file_offset` 翻页避免超大种子一次向浏览器返回全部文件
|
||||
|
||||
生产 Web 页面使用 `/` `/system` 和 `/settings` 三个路由 分别提供搜索运行诊断和配置管理 API 路径继续保持独立避免单页回退冲突
|
||||
|
||||
## 数据恢复
|
||||
|
||||
RocksDB 是权威数据源而 Tantivy 是可重建索引
|
||||
@@ -262,7 +296,7 @@ RocksDB 是权威数据源而 Tantivy 是可重建索引
|
||||
|
||||
项目当前处于开发阶段 持久化结构变化时直接清理测试数据重新采集 不维护旧测试数据库兼容层
|
||||
|
||||
正常退出会先停止 DHT 再排空持久化队列提交剩余索引最后关闭 HTTP 服务
|
||||
正常退出会先停止 DHT 和诊断采样 再排空持久化队列提交剩余索引最后关闭 HTTP 服务
|
||||
|
||||
## 规模基准
|
||||
|
||||
@@ -1,27 +1,32 @@
|
||||
// 负责处理搜索详情统计和健康检查请求
|
||||
|
||||
use std::str::FromStr;
|
||||
use std::{
|
||||
str::FromStr,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
domain::{InfoHash, MetadataRejectionReason},
|
||||
search::{SearchMode, SearchOptions, SearchPage, SearchSort},
|
||||
storage::VerificationPriority,
|
||||
};
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use dht_search::{
|
||||
domain::{InfoHash, MetadataRejectionReason},
|
||||
search::{SearchMode, SearchOptions, SearchPage, SearchSort},
|
||||
storage::VerificationPriority,
|
||||
};
|
||||
|
||||
use super::{
|
||||
ApiState,
|
||||
request::{ContentVariantsRequest, SearchRequest, TorrentRequest},
|
||||
request::{ContentVariantsRequest, DiagnosticHistoryRequest, SearchRequest, TorrentRequest},
|
||||
response::{
|
||||
ContentVariantsResponse, ErrorResponse, StatsResponse, StatusResponse, TorrentResponse,
|
||||
TorrentVariantResponse,
|
||||
},
|
||||
};
|
||||
use crate::config::{ConfigServiceError, ConfigSnapshot, ConfigUpdateRequest};
|
||||
use crate::diagnostics::{CurrentDiagnosticsResponse, DiagnosticHistory, HistoryResolution};
|
||||
|
||||
pub(crate) async fn health() -> Json<StatusResponse> {
|
||||
Json(StatusResponse { status: "ok" })
|
||||
@@ -37,12 +42,20 @@ pub(crate) async fn stats(State(state): State<ApiState>) -> Json<StatsResponse>
|
||||
let persistence = state.persistence.snapshot();
|
||||
let disk = state.disk_guard.snapshot();
|
||||
let backup = state.backup_stats.snapshot();
|
||||
let http = state.http_stats.snapshot();
|
||||
let filtered = persistence.filtered;
|
||||
let verification = state
|
||||
.verification
|
||||
.as_ref()
|
||||
.map(|ingress| ingress.stats().snapshot());
|
||||
Json(StatsResponse {
|
||||
http_active_requests: http.active_requests,
|
||||
http_requests: http.requests,
|
||||
http_client_errors: http.client_errors,
|
||||
http_server_errors: http.server_errors,
|
||||
http_latency_average_micros: http.latency_average_micros,
|
||||
http_latency_p95_millis: http.latency_p95_millis,
|
||||
http_latency_max_millis: http.latency_max_millis,
|
||||
backup_created: backup.created,
|
||||
backup_failed: backup.failed,
|
||||
backup_skipped: backup.skipped,
|
||||
@@ -128,6 +141,58 @@ pub(crate) async fn stats(State(state): State<ApiState>) -> Json<StatsResponse>
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn diagnostics_current(
|
||||
State(state): State<ApiState>,
|
||||
) -> Json<CurrentDiagnosticsResponse> {
|
||||
Json(state.diagnostics.current())
|
||||
}
|
||||
|
||||
pub(crate) async fn diagnostics_history(
|
||||
State(state): State<ApiState>,
|
||||
Query(request): Query<DiagnosticHistoryRequest>,
|
||||
) -> Result<Json<DiagnosticHistory>, ApiError> {
|
||||
const MAX_RANGE_SECS: u64 = 31 * 86_400;
|
||||
if request.range_secs == 0 || request.range_secs > MAX_RANGE_SECS {
|
||||
return Err(ApiError::bad_request("range_secs 必须在 1 到 2678400 之间"));
|
||||
}
|
||||
let resolution = match request.resolution.as_str() {
|
||||
"raw" => HistoryResolution::Raw,
|
||||
"minute" => HistoryResolution::Minute,
|
||||
_ => return Err(ApiError::bad_request("resolution 只支持 raw 或 minute")),
|
||||
};
|
||||
let to = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let from = to.saturating_sub(request.range_secs);
|
||||
let diagnostics = state.diagnostics.clone();
|
||||
let history = tokio::task::spawn_blocking(move || diagnostics.history(resolution, from, to))
|
||||
.await
|
||||
.map_err(|error| ApiError::internal(format!("诊断历史查询任务失败: {error}")))?
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
Ok(Json(history))
|
||||
}
|
||||
|
||||
pub(crate) async fn config_snapshot(State(state): State<ApiState>) -> Json<ConfigSnapshot> {
|
||||
Json(state.config.snapshot())
|
||||
}
|
||||
|
||||
pub(crate) async fn config_update(
|
||||
State(state): State<ApiState>,
|
||||
Json(request): Json<ConfigUpdateRequest>,
|
||||
) -> Result<Json<ConfigSnapshot>, ApiError> {
|
||||
let config = state.config.clone();
|
||||
let snapshot = tokio::task::spawn_blocking(move || config.update(request))
|
||||
.await
|
||||
.map_err(|error| ApiError::internal(format!("配置保存任务失败: {error}")))?
|
||||
.map_err(|error| match error {
|
||||
ConfigServiceError::Conflict => ApiError::conflict(error.to_string()),
|
||||
ConfigServiceError::Validation(_) => ApiError::bad_request(error.to_string()),
|
||||
ConfigServiceError::Persistence(_) => ApiError::internal(error.to_string()),
|
||||
})?;
|
||||
Ok(Json(snapshot))
|
||||
}
|
||||
|
||||
pub(crate) async fn search(
|
||||
State(state): State<ApiState>,
|
||||
Query(request): Query<SearchRequest>,
|
||||
@@ -315,6 +380,13 @@ impl ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
fn conflict(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
status: StatusCode::CONFLICT,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn internal(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
status: StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -6,14 +6,24 @@ mod response;
|
||||
|
||||
use std::{net::SocketAddr, path::PathBuf, sync::Arc};
|
||||
|
||||
use axum::{Router, routing::get};
|
||||
use crate::{search::SearchEngine, storage::TorrentRepository};
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Request, State},
|
||||
middleware::{self, Next},
|
||||
response::Response,
|
||||
routing::get,
|
||||
};
|
||||
use dht_crawler::DhtRuntimeStats;
|
||||
use dht_search::{search::SearchEngine, storage::TorrentRepository};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
|
||||
use crate::{
|
||||
backup::BackupStats, crawler::pipeline::PersistenceIngress, disk_guard::DiskGuard,
|
||||
backup::BackupStats,
|
||||
config::ConfigService,
|
||||
crawler::pipeline::PersistenceIngress,
|
||||
diagnostics::{DiagnosticsHandle, HttpStats},
|
||||
disk_guard::DiskGuard,
|
||||
verification::VerificationIngress,
|
||||
};
|
||||
|
||||
@@ -26,6 +36,9 @@ pub(crate) struct ApiState {
|
||||
pub(crate) verification: Option<VerificationIngress>,
|
||||
pub(crate) disk_guard: DiskGuard,
|
||||
pub(crate) backup_stats: BackupStats,
|
||||
pub(crate) diagnostics: DiagnosticsHandle,
|
||||
pub(crate) config: ConfigService,
|
||||
pub(crate) http_stats: HttpStats,
|
||||
}
|
||||
|
||||
pub(crate) async fn serve(
|
||||
@@ -44,36 +57,54 @@ pub(crate) async fn serve(
|
||||
|
||||
fn router(state: ApiState, web_dir: PathBuf) -> Router {
|
||||
let index = web_dir.join("index.html");
|
||||
let http_stats = state.http_stats.clone();
|
||||
Router::new()
|
||||
.route("/health", get(handlers::health))
|
||||
.route("/ready", get(handlers::ready))
|
||||
.route("/stats", get(handlers::stats))
|
||||
.route("/diagnostics/current", get(handlers::diagnostics_current))
|
||||
.route("/diagnostics/history", get(handlers::diagnostics_history))
|
||||
.route(
|
||||
"/config",
|
||||
get(handlers::config_snapshot).put(handlers::config_update),
|
||||
)
|
||||
.route("/search", get(handlers::search))
|
||||
.route("/contents/{content_key}", get(handlers::content_variants))
|
||||
.route("/torrents/{info_hash}", get(handlers::torrent))
|
||||
.with_state(state)
|
||||
.fallback_service(ServeDir::new(web_dir).fallback(ServeFile::new(index)))
|
||||
.layer(middleware::from_fn_with_state(http_stats, observe_http))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
async fn observe_http(State(stats): State<HttpStats>, request: Request, next: Next) -> Response {
|
||||
let timer = stats.begin();
|
||||
let response = next.run(request).await;
|
||||
timer.finish(response.status().as_u16());
|
||||
response
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{
|
||||
domain::{InfoHash, MetadataCandidate, MetadataLimits, TorrentFile, TorrentRecord},
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use dht_crawler::{DhtRuntimeStats, FileInfo, TorrentInfo};
|
||||
use dht_search::{
|
||||
domain::{InfoHash, MetadataLimits, TorrentRecord},
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
use dht_crawler::DhtRuntimeStats;
|
||||
use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::{
|
||||
config::DiskGuardConfig, crawler::pipeline::PersistencePipeline, disk_guard::DiskGuard,
|
||||
config::{AppConfigDto, ConfigService, DiskGuardConfig, TomlConfigStore},
|
||||
crawler::pipeline::PersistencePipeline,
|
||||
diagnostics::DiagnosticsRuntime,
|
||||
disk_guard::DiskGuard,
|
||||
verification::VerificationIngress,
|
||||
};
|
||||
|
||||
@@ -84,13 +115,12 @@ mod tests {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let repository =
|
||||
Arc::new(RocksTorrentRepository::open(directory.path().join("rocksdb")).unwrap());
|
||||
let mut record = TorrentRecord::try_from(TorrentInfo {
|
||||
let mut record = TorrentRecord::try_from(MetadataCandidate {
|
||||
info_hash: "0101010101010101010101010101010101010101".into(),
|
||||
magnet_link: String::new(),
|
||||
name: "Example Movie".into(),
|
||||
total_size: 205,
|
||||
files: (0..205)
|
||||
.map(|index| FileInfo {
|
||||
.map(|index| TorrentFile {
|
||||
path: if index == 0 {
|
||||
"movie.mkv".into()
|
||||
} else {
|
||||
@@ -100,11 +130,11 @@ mod tests {
|
||||
})
|
||||
.collect(),
|
||||
piece_length: 16_384,
|
||||
peers: vec!["127.0.0.1:6881".into()],
|
||||
source_peers: vec!["127.0.0.1:6881".into()],
|
||||
timestamp: 10,
|
||||
})
|
||||
.unwrap();
|
||||
record.availability = dht_search::domain::Availability::default();
|
||||
record.availability = crate::domain::Availability::default();
|
||||
repository.upsert(record.clone()).unwrap();
|
||||
let mut variant = record.clone();
|
||||
variant.info_hash = InfoHash::from_bytes([2; 20]);
|
||||
@@ -136,18 +166,28 @@ mod tests {
|
||||
verification: Some(verification),
|
||||
disk_guard,
|
||||
backup_stats: BackupStats::default(),
|
||||
diagnostics: DiagnosticsRuntime::disabled().handle(),
|
||||
config: ConfigService::new(
|
||||
Arc::new(TomlConfigStore::new(directory.path().join("service.toml"))),
|
||||
AppConfigDto::default(),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap(),
|
||||
http_stats: HttpStats::default(),
|
||||
},
|
||||
web_dir,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(body.as_ref(), b"<main>DHT Search</main>");
|
||||
for uri in ["/", "/system", "/settings"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(body.as_ref(), b"<main>DHT Search</main>");
|
||||
}
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
@@ -168,6 +208,95 @@ mod tests {
|
||||
assert_eq!(json["disk_state"], "normal");
|
||||
assert!(json["disk_available_bytes"].is_null());
|
||||
assert_eq!(json["backup_created"], 0);
|
||||
assert_eq!(json["http_active_requests"], 1);
|
||||
assert!(
|
||||
json["http_requests"]
|
||||
.as_u64()
|
||||
.is_some_and(|value| value >= 3)
|
||||
);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/diagnostics/current")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(json["status"]["enabled"], false);
|
||||
assert!(json["sample"].is_null());
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/diagnostics/history?resolution=seconds")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/config")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let mut config_snapshot: serde_json::Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
let original_revision = config_snapshot["revision"].as_str().unwrap().to_owned();
|
||||
config_snapshot["config"]["dht"]["port"] = serde_json::json!(22_313);
|
||||
let update = serde_json::json!({
|
||||
"revision": original_revision,
|
||||
"config": config_snapshot["config"].clone(),
|
||||
});
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("PUT")
|
||||
.uri("/config")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&update).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(json["config"]["dht"]["port"], 22_313);
|
||||
assert_eq!(json["restart_required"], true);
|
||||
assert!(directory.path().join("service.toml").exists());
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("PUT")
|
||||
.uri("/config")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&update).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CONFLICT);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
@@ -1,6 +1,6 @@
|
||||
// 负责定义 HTTP 查询参数和输入校验模型
|
||||
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
domain::{AvailabilityStatus, HeatLevel},
|
||||
search::{SearchMode, SearchSort},
|
||||
};
|
||||
@@ -50,6 +50,22 @@ pub(crate) struct TorrentRequest {
|
||||
pub(crate) file_limit: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct DiagnosticHistoryRequest {
|
||||
#[serde(default = "default_diagnostic_range")]
|
||||
pub(crate) range_secs: u64,
|
||||
#[serde(default = "default_diagnostic_resolution")]
|
||||
pub(crate) resolution: String,
|
||||
}
|
||||
|
||||
fn default_diagnostic_range() -> u64 {
|
||||
3_600
|
||||
}
|
||||
|
||||
fn default_diagnostic_resolution() -> String {
|
||||
"raw".to_owned()
|
||||
}
|
||||
|
||||
fn default_file_limit() -> usize {
|
||||
100
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
// 负责定义稳定的 HTTP 响应模型和领域对象转换边界
|
||||
|
||||
use dht_search::domain::{Availability, Heat, TorrentFile, TorrentRecord};
|
||||
use crate::domain::{Availability, Heat, TorrentFile, TorrentRecord};
|
||||
use serde::Serialize;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
@@ -16,6 +16,13 @@ pub(crate) struct ErrorResponse {
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct StatsResponse {
|
||||
pub(crate) http_active_requests: u64,
|
||||
pub(crate) http_requests: u64,
|
||||
pub(crate) http_client_errors: u64,
|
||||
pub(crate) http_server_errors: u64,
|
||||
pub(crate) http_latency_average_micros: Option<u64>,
|
||||
pub(crate) http_latency_p95_millis: Option<u64>,
|
||||
pub(crate) http_latency_max_millis: u64,
|
||||
pub(crate) backup_created: u64,
|
||||
pub(crate) backup_failed: u64,
|
||||
pub(crate) backup_skipped: u64,
|
||||
@@ -6,25 +6,26 @@ use std::{
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use dht_crawler::DHTServer;
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
domain::InfoHash,
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
use dht_crawler::DHTServer;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::{
|
||||
api::{self, ApiState},
|
||||
backup::{self, BackupStats},
|
||||
config::AppConfig,
|
||||
config::{AppConfig, ConfigService},
|
||||
crawler::pipeline::PersistencePipeline,
|
||||
diagnostics::{DiagnosticSources, DiagnosticsRuntime, HttpStats},
|
||||
disk_guard::{self, DiskGuard},
|
||||
error::AppError,
|
||||
index_worker, monitor, shutdown, verification,
|
||||
};
|
||||
|
||||
pub(crate) async fn run(config: AppConfig) -> Result<(), AppError> {
|
||||
pub(crate) async fn run(config: AppConfig, config_service: ConfigService) -> Result<(), AppError> {
|
||||
std::fs::create_dir_all(&config.data_dir)?;
|
||||
let _data_lock = backup::acquire_data_lock(&config.data_dir)?;
|
||||
let disk_guard = DiskGuard::new(&config.disk_guard);
|
||||
@@ -214,6 +215,28 @@ pub(crate) async fn run(config: AppConfig) -> Result<(), AppError> {
|
||||
index_cancel.clone(),
|
||||
index_fatal_tx,
|
||||
));
|
||||
let http_stats = HttpStats::default();
|
||||
let diagnostics = if config.diagnostics.enabled {
|
||||
match DiagnosticsRuntime::start(
|
||||
config.diagnostics.clone(),
|
||||
DiagnosticSources {
|
||||
repository: repository.clone(),
|
||||
search: search.clone(),
|
||||
dht: server.runtime_stats(),
|
||||
persistence: persistence.ingress.clone(),
|
||||
disk_guard: disk_guard.clone(),
|
||||
http: http_stats.clone(),
|
||||
},
|
||||
) {
|
||||
Ok(runtime) => runtime,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "运行诊断历史初始化失败 将继续提供核心服务");
|
||||
DiagnosticsRuntime::disabled()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DiagnosticsRuntime::disabled()
|
||||
};
|
||||
let api_cancel = CancellationToken::new();
|
||||
let mut api_task = tokio::spawn(api::serve(
|
||||
config.http.listen,
|
||||
@@ -226,6 +249,9 @@ pub(crate) async fn run(config: AppConfig) -> Result<(), AppError> {
|
||||
verification: verification_ingress,
|
||||
disk_guard: disk_guard.clone(),
|
||||
backup_stats,
|
||||
diagnostics: diagnostics.handle(),
|
||||
config: config_service,
|
||||
http_stats,
|
||||
},
|
||||
api_cancel.clone(),
|
||||
));
|
||||
@@ -269,8 +295,11 @@ pub(crate) async fn run(config: AppConfig) -> Result<(), AppError> {
|
||||
}
|
||||
};
|
||||
|
||||
let mut shutdown_error = None;
|
||||
diagnostics.request_shutdown();
|
||||
verification_cancel.cancel();
|
||||
backup_cancel.cancel();
|
||||
server.shutdown();
|
||||
if let Some(task) = backup_task {
|
||||
let _ = task.await;
|
||||
}
|
||||
@@ -278,34 +307,76 @@ pub(crate) async fn run(config: AppConfig) -> Result<(), AppError> {
|
||||
if let Some(task) = verification_task {
|
||||
match task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => return Err(AppError::VerificationWorker(error)),
|
||||
Err(error) => return Err(AppError::VerificationWorker(error.to_string())),
|
||||
Ok(Err(error)) => {
|
||||
remember_shutdown_error(&mut shutdown_error, AppError::VerificationWorker(error));
|
||||
}
|
||||
Err(error) => {
|
||||
remember_shutdown_error(
|
||||
&mut shutdown_error,
|
||||
AppError::VerificationWorker(error.to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
server.shutdown();
|
||||
disk_cancel.cancel();
|
||||
let _ = disk_task.await;
|
||||
monitor_cancel.cancel();
|
||||
let _ = monitor.await;
|
||||
persistence.close_and_join().await?;
|
||||
if let Err(error) = persistence.close_and_join().await {
|
||||
remember_shutdown_error(&mut shutdown_error, error);
|
||||
}
|
||||
index_cancel.cancel();
|
||||
match index_task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => return Err(AppError::IndexWorker(error)),
|
||||
Err(error) => return Err(AppError::IndexWorker(error.to_string())),
|
||||
Ok(Err(error)) => {
|
||||
remember_shutdown_error(&mut shutdown_error, AppError::IndexWorker(error));
|
||||
}
|
||||
Err(error) => {
|
||||
remember_shutdown_error(
|
||||
&mut shutdown_error,
|
||||
AppError::IndexWorker(error.to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Err(error) = diagnostics.shutdown().await {
|
||||
remember_shutdown_error(
|
||||
&mut shutdown_error,
|
||||
AppError::Diagnostics(error.to_string()),
|
||||
);
|
||||
}
|
||||
api_cancel.cancel();
|
||||
if !api_task.is_finished() {
|
||||
match api_task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => return Err(AppError::Io(error)),
|
||||
Err(error) => return Err(AppError::Config(format!("HTTP 服务任务异常: {error}"))),
|
||||
Ok(Err(error)) => {
|
||||
remember_shutdown_error(&mut shutdown_error, AppError::Io(error));
|
||||
}
|
||||
Err(error) => {
|
||||
remember_shutdown_error(
|
||||
&mut shutdown_error,
|
||||
AppError::Config(format!("HTTP 服务任务异常: {error}")),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = shutdown_error {
|
||||
if run_result.is_ok() {
|
||||
return Err(error);
|
||||
}
|
||||
tracing::error!(%error, "关闭阶段发生附加错误");
|
||||
}
|
||||
tracing::info!("dht-search 已安全停止");
|
||||
run_result
|
||||
}
|
||||
|
||||
fn remember_shutdown_error(slot: &mut Option<AppError>, error: AppError) {
|
||||
if slot.is_none() {
|
||||
*slot = Some(error);
|
||||
} else {
|
||||
tracing::error!(%error, "关闭阶段发生附加错误");
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_timestamp() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use dht_search::storage::{CheckpointSummary, RocksTorrentRepository, StorageError};
|
||||
use crate::storage::{CheckpointSummary, RocksTorrentRepository, StorageError};
|
||||
use fs2::FileExt;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -377,24 +377,26 @@ fn unix_timestamp_millis() -> u128 {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use dht_crawler::{FileInfo, TorrentInfo};
|
||||
use dht_search::{domain::TorrentRecord, search::SearchEngine, storage::TorrentRepository};
|
||||
use crate::{
|
||||
domain::{MetadataCandidate, TorrentFile, TorrentRecord},
|
||||
search::SearchEngine,
|
||||
storage::TorrentRepository,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn record(byte: u8, name: &str) -> TorrentRecord {
|
||||
TorrentRecord::try_from(TorrentInfo {
|
||||
TorrentRecord::try_from(MetadataCandidate {
|
||||
info_hash: format!("{byte:02x}").repeat(20),
|
||||
magnet_link: String::new(),
|
||||
name: name.into(),
|
||||
total_size: 42,
|
||||
files: vec![FileInfo {
|
||||
files: vec![TorrentFile {
|
||||
path: format!("{name}.bin"),
|
||||
size: 42,
|
||||
}],
|
||||
piece_length: 16_384,
|
||||
peers: Vec::new(),
|
||||
source_peers: Vec::new(),
|
||||
timestamp: 10,
|
||||
})
|
||||
.unwrap()
|
||||
+5
-7
@@ -2,8 +2,7 @@
|
||||
|
||||
use std::error::Error;
|
||||
|
||||
use dht_crawler::{FileInfo, TorrentInfo};
|
||||
use dht_search::domain::{MetadataLimits, TorrentRecord};
|
||||
use dht_search::domain::{MetadataCandidate, MetadataLimits, TorrentFile, TorrentRecord};
|
||||
|
||||
pub(crate) const BASE_TIMESTAMP: u64 = 1_700_000_000;
|
||||
|
||||
@@ -15,12 +14,12 @@ pub(crate) fn generate_record(
|
||||
let file_count = 1 + content_id % 4;
|
||||
let mut files = Vec::with_capacity(file_count);
|
||||
let main_size = 64 * 1024 * 1024 + (content_id as u64 % 8_192) * 1_048_576;
|
||||
files.push(FileInfo {
|
||||
files.push(TorrentFile {
|
||||
path: format!("media/category_{}/item_{content_id}.mkv", content_id % 100),
|
||||
size: main_size,
|
||||
});
|
||||
for part in 1..file_count {
|
||||
files.push(FileInfo {
|
||||
files.push(TorrentFile {
|
||||
path: format!("docs/item_{content_id}/part_{part}.txt"),
|
||||
size: 1_024 + (content_id as u64 + part as u64) % 65_536,
|
||||
});
|
||||
@@ -39,14 +38,13 @@ pub(crate) fn generate_record(
|
||||
let digest = blake3::hash(&(index as u64).to_be_bytes());
|
||||
let info_hash = hex::encode(&digest.as_bytes()[..20]);
|
||||
TorrentRecord::try_from_with_limits(
|
||||
TorrentInfo {
|
||||
MetadataCandidate {
|
||||
info_hash,
|
||||
magnet_link: String::new(),
|
||||
name,
|
||||
total_size,
|
||||
files,
|
||||
piece_length: 16_384,
|
||||
peers: Vec::new(),
|
||||
source_peers: Vec::new(),
|
||||
timestamp: BASE_TIMESTAMP.saturating_add(index as u64),
|
||||
},
|
||||
MetadataLimits::default(),
|
||||
@@ -0,0 +1,222 @@
|
||||
// 负责组合配置 DTO 持久化读取命令行覆盖和运行时解析
|
||||
|
||||
mod model;
|
||||
mod runtime;
|
||||
mod service;
|
||||
mod store;
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
use crate::error::AppError;
|
||||
|
||||
pub(crate) use model::{
|
||||
AppConfigDto, BackupConfig, DiagnosticsConfig, DiskGuardConfig, LogRotation, LoggingConfig,
|
||||
VerificationConfig,
|
||||
};
|
||||
pub(crate) use runtime::AppConfig;
|
||||
pub(crate) use service::{ConfigService, ConfigServiceError, ConfigSnapshot, ConfigUpdateRequest};
|
||||
pub(crate) use store::{ConfigStore, TomlConfigStore};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "dht-search", version, about = "DHT 元数据采集和搜索服务")]
|
||||
pub(crate) struct Cli {
|
||||
#[arg(long, default_value = "dht-search.toml")]
|
||||
config: PathBuf,
|
||||
#[arg(long)]
|
||||
data_dir: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
run_duration_secs: Option<u64>,
|
||||
#[arg(long)]
|
||||
restore_checkpoint: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct StartupConfig {
|
||||
pub(crate) app: AppConfig,
|
||||
pub(crate) config_service: ConfigService,
|
||||
pub(crate) restore_checkpoint: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub(crate) fn load(self) -> Result<StartupConfig, AppError> {
|
||||
let config_path = absolute_path(&self.config)?;
|
||||
let store = Arc::new(TomlConfigStore::new(config_path.clone()));
|
||||
let persisted = store.load()?.unwrap_or_default();
|
||||
let mut dto = persisted.clone();
|
||||
let mut command_line_overrides = Vec::new();
|
||||
|
||||
if let Some(data_dir) = self.data_dir {
|
||||
dto.data_dir = data_dir;
|
||||
command_line_overrides.push("data_dir".to_owned());
|
||||
}
|
||||
if self.run_duration_secs.is_some() {
|
||||
dto.run_duration_secs = self.run_duration_secs;
|
||||
command_line_overrides.push("run_duration_secs".to_owned());
|
||||
}
|
||||
let base = config_path
|
||||
.parent()
|
||||
.ok_or_else(|| AppError::Config("配置文件没有父目录".to_owned()))?;
|
||||
let app = AppConfig::resolve(dto, base)?;
|
||||
let config_service = ConfigService::new(store, persisted, command_line_overrides)?;
|
||||
let restore_checkpoint = self
|
||||
.restore_checkpoint
|
||||
.map(|path| absolute_path(&path))
|
||||
.transpose()?;
|
||||
Ok(StartupConfig {
|
||||
app,
|
||||
config_service,
|
||||
restore_checkpoint,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn absolute_path(path: &Path) -> Result<PathBuf, AppError> {
|
||||
if path.is_absolute() {
|
||||
Ok(path.to_path_buf())
|
||||
} else {
|
||||
Ok(std::env::current_dir()?.join(path))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn resolve(dto: AppConfigDto) -> Result<AppConfig, AppError> {
|
||||
AppConfig::resolve(dto, Path::new("C:/config-root"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_produce_valid_dht_options() {
|
||||
let config = resolve(AppConfigDto::default()).unwrap();
|
||||
let options = config.dht_options();
|
||||
assert_eq!(options.port, 12_313);
|
||||
assert_eq!(options.metadata.max_worker_count, 400);
|
||||
assert_eq!(options.metadata.max_connects_per_second, 400);
|
||||
assert_eq!(options.metadata.max_metadata_size_bytes, 10 * 1024 * 1024);
|
||||
assert_eq!(options.max_outbound_queries_per_second, 1_000);
|
||||
assert_eq!(options.sample_infohashes.max_queries_per_second, 60);
|
||||
assert_eq!(options.sample_infohashes.max_in_flight, 100);
|
||||
assert_eq!(options.sample_infohashes.new_node_sample_percent, 50);
|
||||
assert_eq!(options.sample_infohashes.candidate_queue_capacity, 8_192);
|
||||
assert!(!options.sample_infohashes.fallback_to_iterative);
|
||||
assert_eq!(options.peer_lookup.max_lookups_per_second, 200);
|
||||
assert_eq!(options.peer_lookup.max_active_lookups, 200);
|
||||
assert_eq!(options.crawl.rate_limit.max_find_node_rate_per_sec, 10);
|
||||
assert_eq!(
|
||||
options.crawl.rate_limit.max_new_destinations_per_minute,
|
||||
12_000
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dto_round_trips_through_toml() {
|
||||
let dto = AppConfigDto::default();
|
||||
let encoded = toml::to_string(&dto).unwrap();
|
||||
let decoded: AppConfigDto = toml::from_str(&encoded).unwrap();
|
||||
assert_eq!(decoded.data_dir, dto.data_dir);
|
||||
assert_eq!(decoded.dht.port, dto.dht.port);
|
||||
assert_eq!(decoded.http.listen, dto.http.listen);
|
||||
assert_eq!(decoded.logging.file_prefix, dto.logging.file_prefix);
|
||||
assert_eq!(decoded.diagnostics.database, dto.diagnostics.database);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_directories_are_resolved_from_config_file() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let config_path = directory.path().join("service.toml");
|
||||
fs::write(&config_path, "data_dir = 'state'").unwrap();
|
||||
let startup = Cli {
|
||||
config: config_path,
|
||||
data_dir: None,
|
||||
run_duration_secs: None,
|
||||
restore_checkpoint: None,
|
||||
}
|
||||
.load()
|
||||
.unwrap();
|
||||
let config = startup.app;
|
||||
assert_eq!(config.data_dir, directory.path().join("state"));
|
||||
assert_eq!(
|
||||
config.content_filter_file,
|
||||
directory.path().join("content-filters.toml")
|
||||
);
|
||||
assert_eq!(config.logging.directory, directory.path().join("data/logs"));
|
||||
assert_eq!(
|
||||
config.backup.directory,
|
||||
directory.path().join("data/backups")
|
||||
);
|
||||
assert_eq!(
|
||||
config.diagnostics.database,
|
||||
directory.path().join("data/diagnostics.sqlite3")
|
||||
);
|
||||
assert_eq!(config.http.web_dir, directory.path().join("src/web/dist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_config_field_is_rejected() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let config_path = directory.path().join("service.toml");
|
||||
fs::write(&config_path, "unknown = true").unwrap();
|
||||
let error = Cli {
|
||||
config: config_path,
|
||||
data_dir: None,
|
||||
run_duration_secs: None,
|
||||
restore_checkpoint: None,
|
||||
}
|
||||
.load()
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, AppError::Toml(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_metadata_limit_is_rejected() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.metadata_limits.max_files = 0;
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_new_node_sample_percent_is_rejected() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.dht.sample_new_node_percent = 101;
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_resume_threshold_must_exceed_minimum() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.disk_guard.resume_free_bytes = dto.disk_guard.minimum_free_bytes;
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_output() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.logging.file_enabled = false;
|
||||
dto.logging.console_enabled = false;
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_prefix_cannot_escape_the_log_directory() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.logging.file_prefix = "../service".into();
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_directory_cannot_be_inside_rocksdb() {
|
||||
let mut dto = AppConfigDto::default();
|
||||
dto.backup.directory = dto.data_dir.join("rocksdb/checkpoints");
|
||||
assert!(matches!(resolve(dto), Err(AppError::Config(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
// 负责定义可序列化的用户配置 DTO 和默认值
|
||||
|
||||
use std::{net::SocketAddr, path::PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct AppConfigDto {
|
||||
pub(crate) data_dir: PathBuf,
|
||||
pub(crate) content_filter_file: PathBuf,
|
||||
pub(crate) persistence_queue_capacity: usize,
|
||||
pub(crate) stats_interval_secs: u64,
|
||||
pub(crate) run_duration_secs: Option<u64>,
|
||||
pub(crate) index_batch_size: usize,
|
||||
pub(crate) index_interval_millis: u64,
|
||||
pub(crate) metadata_limits: MetadataLimitsConfig,
|
||||
pub(crate) dht: DhtConfig,
|
||||
pub(crate) disk_guard: DiskGuardConfig,
|
||||
pub(crate) backup: BackupConfig,
|
||||
pub(crate) diagnostics: DiagnosticsConfig,
|
||||
pub(crate) logging: LoggingConfig,
|
||||
pub(crate) http: HttpConfig,
|
||||
pub(crate) verification: VerificationConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct DhtConfig {
|
||||
pub(crate) port: u16,
|
||||
pub(crate) netmode: NetworkMode,
|
||||
pub(crate) hash_queue_capacity: usize,
|
||||
pub(crate) max_outbound_queries_per_second: u32,
|
||||
pub(crate) outbound_query_burst: u32,
|
||||
pub(crate) metadata_timeout_secs: u64,
|
||||
pub(crate) metadata_queue_capacity: usize,
|
||||
pub(crate) metadata_workers: usize,
|
||||
pub(crate) metadata_connects_per_second: u32,
|
||||
pub(crate) sample_queries_per_second: u32,
|
||||
pub(crate) sample_max_in_flight: usize,
|
||||
pub(crate) sample_new_node_percent: u8,
|
||||
pub(crate) sample_candidate_queue_capacity: usize,
|
||||
pub(crate) sample_fallback_to_iterative: bool,
|
||||
pub(crate) peer_lookups_per_second: u32,
|
||||
pub(crate) peer_lookup_max_active: usize,
|
||||
pub(crate) find_node_queries_per_second: u32,
|
||||
pub(crate) find_node_max_in_flight: usize,
|
||||
pub(crate) new_destinations_per_minute: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct HttpConfig {
|
||||
pub(crate) listen: SocketAddr,
|
||||
pub(crate) web_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct DiskGuardConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) check_interval_secs: u64,
|
||||
pub(crate) minimum_free_bytes: u64,
|
||||
pub(crate) resume_free_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct LoggingConfig {
|
||||
pub(crate) directory: PathBuf,
|
||||
pub(crate) file_enabled: bool,
|
||||
pub(crate) console_enabled: bool,
|
||||
pub(crate) rotation: LogRotation,
|
||||
pub(crate) retain_files: usize,
|
||||
pub(crate) file_prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) directory: PathBuf,
|
||||
pub(crate) interval_secs: u64,
|
||||
pub(crate) retain_checkpoints: usize,
|
||||
pub(crate) create_on_start: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct DiagnosticsConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) database: PathBuf,
|
||||
pub(crate) sample_interval_secs: u64,
|
||||
pub(crate) raw_retention_hours: u64,
|
||||
pub(crate) minute_retention_days: u64,
|
||||
pub(crate) queue_capacity: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum LogRotation {
|
||||
Minutely,
|
||||
Hourly,
|
||||
#[default]
|
||||
Daily,
|
||||
Never,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct VerificationConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) queue_capacity: usize,
|
||||
pub(crate) max_active: usize,
|
||||
pub(crate) max_peer_attempts: usize,
|
||||
pub(crate) lease_secs: u64,
|
||||
pub(crate) poll_interval_millis: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub(crate) struct MetadataLimitsConfig {
|
||||
pub(crate) max_metadata_bytes: usize,
|
||||
pub(crate) max_files: usize,
|
||||
pub(crate) max_name_bytes: usize,
|
||||
pub(crate) max_path_bytes: usize,
|
||||
pub(crate) max_path_depth: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum NetworkMode {
|
||||
#[default]
|
||||
Ipv4Only,
|
||||
Ipv6Only,
|
||||
DualStack,
|
||||
}
|
||||
|
||||
impl Default for AppConfigDto {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
data_dir: PathBuf::from("data"),
|
||||
content_filter_file: PathBuf::from("content-filters.toml"),
|
||||
persistence_queue_capacity: 8_192,
|
||||
stats_interval_secs: 10,
|
||||
run_duration_secs: None,
|
||||
index_batch_size: 1_024,
|
||||
index_interval_millis: 5_000,
|
||||
metadata_limits: MetadataLimitsConfig::default(),
|
||||
dht: DhtConfig::default(),
|
||||
disk_guard: DiskGuardConfig::default(),
|
||||
backup: BackupConfig::default(),
|
||||
diagnostics: DiagnosticsConfig::default(),
|
||||
logging: LoggingConfig::default(),
|
||||
http: HttpConfig::default(),
|
||||
verification: VerificationConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MetadataLimitsConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_metadata_bytes: 10 * 1024 * 1024,
|
||||
max_files: 20_000,
|
||||
max_name_bytes: 1_024,
|
||||
max_path_bytes: 4_096,
|
||||
max_path_depth: 64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DhtConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
port: 12_313,
|
||||
netmode: NetworkMode::Ipv4Only,
|
||||
hash_queue_capacity: 20_000,
|
||||
max_outbound_queries_per_second: 1_000,
|
||||
outbound_query_burst: 200,
|
||||
metadata_timeout_secs: 6,
|
||||
metadata_queue_capacity: 20_000,
|
||||
metadata_workers: 400,
|
||||
metadata_connects_per_second: 400,
|
||||
sample_queries_per_second: 60,
|
||||
sample_max_in_flight: 100,
|
||||
sample_new_node_percent: 50,
|
||||
sample_candidate_queue_capacity: 8_192,
|
||||
sample_fallback_to_iterative: false,
|
||||
peer_lookups_per_second: 200,
|
||||
peer_lookup_max_active: 200,
|
||||
find_node_queries_per_second: 10,
|
||||
find_node_max_in_flight: 100,
|
||||
new_destinations_per_minute: 12_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HttpConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
listen: SocketAddr::from(([127, 0, 0, 1], 8080)),
|
||||
web_dir: PathBuf::from("src/web/dist"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiskGuardConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
check_interval_secs: 10,
|
||||
minimum_free_bytes: 5 * 1024 * 1024 * 1024,
|
||||
resume_free_bytes: 6 * 1024 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
directory: PathBuf::from("data/logs"),
|
||||
file_enabled: true,
|
||||
console_enabled: false,
|
||||
rotation: LogRotation::Daily,
|
||||
retain_files: 7,
|
||||
file_prefix: "dht-search".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BackupConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
directory: PathBuf::from("data/backups"),
|
||||
interval_secs: 6 * 60 * 60,
|
||||
retain_checkpoints: 3,
|
||||
create_on_start: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiagnosticsConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
database: PathBuf::from("data/diagnostics.sqlite3"),
|
||||
sample_interval_secs: 10,
|
||||
raw_retention_hours: 24,
|
||||
minute_retention_days: 30,
|
||||
queue_capacity: 128,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for VerificationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
queue_capacity: 10_000,
|
||||
max_active: 8,
|
||||
max_peer_attempts: 3,
|
||||
lease_secs: 60,
|
||||
poll_interval_millis: 250,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// 负责解析校验用户配置并生成应用可直接使用的运行时配置
|
||||
|
||||
use std::{fs, ops::Deref, path::Path};
|
||||
|
||||
use crate::domain::{ContentFilter, ContentFilterConfig, MetadataLimits};
|
||||
use dht_crawler::{
|
||||
BootstrapOptions, CrawlOptions, DHTOptions, MetadataOptions, NetMode, PeerLookupOptions,
|
||||
PoolOptions, RateLimitOptions, SampleInfohashesOptions, SchedulerOptions, TargetOptions,
|
||||
};
|
||||
|
||||
use crate::error::AppError;
|
||||
|
||||
use super::model::{AppConfigDto, NetworkMode};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct AppConfig(AppConfigDto);
|
||||
|
||||
impl AppConfig {
|
||||
pub(crate) fn resolve(mut dto: AppConfigDto, base: &Path) -> Result<Self, AppError> {
|
||||
dto.data_dir = resolve_path(base, dto.data_dir);
|
||||
dto.content_filter_file = resolve_path(base, dto.content_filter_file);
|
||||
dto.logging.directory = resolve_path(base, dto.logging.directory);
|
||||
dto.backup.directory = resolve_path(base, dto.backup.directory);
|
||||
dto.diagnostics.database = resolve_path(base, dto.diagnostics.database);
|
||||
dto.http.web_dir = resolve_path(base, dto.http.web_dir);
|
||||
let config = Self(dto);
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
pub(crate) fn content_filter(&self) -> Result<ContentFilter, AppError> {
|
||||
let contents = fs::read_to_string(&self.content_filter_file)?;
|
||||
let config = toml::from_str::<ContentFilterConfig>(&contents)?;
|
||||
ContentFilter::compile(config).map_err(AppError::from)
|
||||
}
|
||||
|
||||
pub(crate) fn dht_options(&self) -> DHTOptions {
|
||||
let defaults = DHTOptions::default();
|
||||
DHTOptions {
|
||||
port: self.dht.port,
|
||||
netmode: self.dht.netmode.into(),
|
||||
hash_queue_capacity: self.dht.hash_queue_capacity,
|
||||
max_outbound_queries_per_second: self.dht.max_outbound_queries_per_second,
|
||||
outbound_query_burst: self.dht.outbound_query_burst,
|
||||
metadata: MetadataOptions {
|
||||
timeout_secs: self.dht.metadata_timeout_secs,
|
||||
max_queue_size: self.dht.metadata_queue_capacity,
|
||||
max_worker_count: self.dht.metadata_workers,
|
||||
max_connects_per_second: self.dht.metadata_connects_per_second,
|
||||
max_metadata_size_bytes: self.metadata_limits.max_metadata_bytes,
|
||||
..defaults.metadata
|
||||
},
|
||||
peer_lookup: PeerLookupOptions {
|
||||
max_lookups_per_second: self.dht.peer_lookups_per_second,
|
||||
burst: self.dht.peer_lookups_per_second.max(1),
|
||||
max_active_lookups: self.dht.peer_lookup_max_active,
|
||||
},
|
||||
sample_infohashes: SampleInfohashesOptions {
|
||||
max_queries_per_second: self.dht.sample_queries_per_second,
|
||||
burst: self.dht.sample_queries_per_second.max(1),
|
||||
max_in_flight: self.dht.sample_max_in_flight,
|
||||
new_node_sample_percent: self.dht.sample_new_node_percent,
|
||||
candidate_queue_capacity: self.dht.sample_candidate_queue_capacity,
|
||||
fallback_to_iterative: self.dht.sample_fallback_to_iterative,
|
||||
..defaults.sample_infohashes
|
||||
},
|
||||
crawl: CrawlOptions {
|
||||
pool: PoolOptions {
|
||||
..defaults.crawl.pool
|
||||
},
|
||||
rate_limit: RateLimitOptions {
|
||||
max_find_node_rate_per_sec: self.dht.find_node_queries_per_second,
|
||||
burst: self.dht.outbound_query_burst,
|
||||
max_in_flight: self.dht.find_node_max_in_flight,
|
||||
max_new_destinations_per_minute: self.dht.new_destinations_per_minute,
|
||||
..defaults.crawl.rate_limit
|
||||
},
|
||||
bootstrap: BootstrapOptions {
|
||||
..defaults.crawl.bootstrap
|
||||
},
|
||||
target: TargetOptions {
|
||||
..defaults.crawl.target
|
||||
},
|
||||
scheduler: SchedulerOptions {
|
||||
..defaults.crawl.scheduler
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metadata_limits(&self) -> MetadataLimits {
|
||||
MetadataLimits {
|
||||
max_files: self.metadata_limits.max_files,
|
||||
max_name_bytes: self.metadata_limits.max_name_bytes,
|
||||
max_path_bytes: self.metadata_limits.max_path_bytes,
|
||||
max_path_depth: self.metadata_limits.max_path_depth,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<(), AppError> {
|
||||
if self.persistence_queue_capacity == 0 {
|
||||
return Err(AppError::Config(
|
||||
"persistence_queue_capacity 必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.stats_interval_secs == 0 {
|
||||
return Err(AppError::Config(
|
||||
"stats_interval_secs 必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.run_duration_secs == Some(0) {
|
||||
return Err(AppError::Config(
|
||||
"run_duration_secs 必须大于零或不设置".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.index_batch_size == 0 || self.index_interval_millis == 0 {
|
||||
return Err(AppError::Config(
|
||||
"索引批量大小和执行间隔必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.metadata_limits.max_metadata_bytes == 0
|
||||
|| self.metadata_limits.max_files == 0
|
||||
|| self.metadata_limits.max_name_bytes == 0
|
||||
|| self.metadata_limits.max_path_bytes == 0
|
||||
|| self.metadata_limits.max_path_depth == 0
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"Metadata 大小文件数名称路径和目录层级上限必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.dht.metadata_workers == 0 || self.dht.metadata_queue_capacity == 0 {
|
||||
return Err(AppError::Config(
|
||||
"Metadata worker 和队列容量必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.verification.queue_capacity == 0
|
||||
|| self.verification.max_active == 0
|
||||
|| self.verification.max_peer_attempts == 0
|
||||
|| self.verification.lease_secs == 0
|
||||
|| self.verification.poll_interval_millis == 0
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"验证队列容量并发尝试数租约和轮询间隔必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.disk_guard.check_interval_secs == 0
|
||||
|| self.disk_guard.minimum_free_bytes == 0
|
||||
|| self.disk_guard.resume_free_bytes <= self.disk_guard.minimum_free_bytes
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"磁盘检查间隔必须大于零且恢复阈值必须大于保护阈值".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.backup.interval_secs == 0 || self.backup.retain_checkpoints == 0 {
|
||||
return Err(AppError::Config(
|
||||
"备份间隔和检查点保留数量必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.diagnostics.sample_interval_secs == 0
|
||||
|| self.diagnostics.raw_retention_hours == 0
|
||||
|| self.diagnostics.minute_retention_days == 0
|
||||
|| self.diagnostics.queue_capacity == 0
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"诊断采样间隔保留时间和队列容量必须大于零".to_owned(),
|
||||
));
|
||||
}
|
||||
let database_path = self.data_dir.join("rocksdb");
|
||||
if self.backup.directory == database_path
|
||||
|| self.backup.directory.starts_with(&database_path)
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"检查点目录不能位于 RocksDB 数据库目录内部".to_owned(),
|
||||
));
|
||||
}
|
||||
if !self.logging.file_enabled && !self.logging.console_enabled {
|
||||
return Err(AppError::Config(
|
||||
"文件日志和终端日志不能同时关闭".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.logging.file_enabled && self.logging.retain_files == 0 {
|
||||
return Err(AppError::Config("日志保留文件数量必须大于零".to_owned()));
|
||||
}
|
||||
if self.logging.file_prefix.trim().is_empty()
|
||||
|| self
|
||||
.logging
|
||||
.file_prefix
|
||||
.chars()
|
||||
.any(|character| character.is_control() || matches!(character, '/' | '\\'))
|
||||
{
|
||||
return Err(AppError::Config(
|
||||
"日志文件前缀不能为空且不能包含路径分隔符或控制字符".to_owned(),
|
||||
));
|
||||
}
|
||||
if self.dht.max_outbound_queries_per_second == 0
|
||||
|| self.dht.outbound_query_burst == 0
|
||||
|| self.dht.metadata_connects_per_second == 0
|
||||
|| self.dht.sample_max_in_flight == 0
|
||||
|| self.dht.sample_candidate_queue_capacity == 0
|
||||
|| self.dht.peer_lookup_max_active == 0
|
||||
|| self.dht.find_node_max_in_flight == 0
|
||||
{
|
||||
return Err(AppError::Config("网络速率和并发上限必须大于零".to_owned()));
|
||||
}
|
||||
if self.dht.sample_new_node_percent > 100 {
|
||||
return Err(AppError::Config(
|
||||
"sample_new_node_percent 必须在 0 到 100 之间".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for AppConfig {
|
||||
type Target = AppConfigDto;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NetworkMode> for NetMode {
|
||||
fn from(value: NetworkMode) -> Self {
|
||||
match value {
|
||||
NetworkMode::Ipv4Only => Self::Ipv4Only,
|
||||
NetworkMode::Ipv6Only => Self::Ipv6Only,
|
||||
NetworkMode::DualStack => Self::DualStack,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_path(base: &Path, path: std::path::PathBuf) -> std::path::PathBuf {
|
||||
if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
base.join(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
// 负责配置修订查询完整校验并发覆盖保护和持久化更新
|
||||
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::AppError;
|
||||
|
||||
use super::{AppConfig, AppConfigDto, ConfigStore};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct ConfigSnapshot {
|
||||
pub(crate) revision: String,
|
||||
pub(crate) restart_required: bool,
|
||||
pub(crate) source: String,
|
||||
pub(crate) command_line_overrides: Vec<String>,
|
||||
pub(crate) config: AppConfigDto,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ConfigUpdateRequest {
|
||||
pub(crate) revision: String,
|
||||
pub(crate) config: AppConfigDto,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ConfigService {
|
||||
inner: Arc<ConfigServiceInner>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ConfigService {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ConfigService")
|
||||
.field("snapshot", &self.snapshot())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
struct ConfigServiceInner {
|
||||
store: Arc<dyn ConfigStore>,
|
||||
base: PathBuf,
|
||||
startup: AppConfigDto,
|
||||
command_line_overrides: Vec<String>,
|
||||
state: Mutex<ConfigState>,
|
||||
}
|
||||
|
||||
struct ConfigState {
|
||||
revision: String,
|
||||
config: AppConfigDto,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum ConfigServiceError {
|
||||
#[error("配置已经被其他请求修改 请重新加载后再保存")]
|
||||
Conflict,
|
||||
#[error("配置校验失败: {0}")]
|
||||
Validation(String),
|
||||
#[error("配置保存失败: {0}")]
|
||||
Persistence(String),
|
||||
}
|
||||
|
||||
impl ConfigService {
|
||||
pub(crate) fn new(
|
||||
store: Arc<dyn ConfigStore>,
|
||||
initial: AppConfigDto,
|
||||
command_line_overrides: Vec<String>,
|
||||
) -> Result<Self, AppError> {
|
||||
let base = store
|
||||
.path()
|
||||
.parent()
|
||||
.ok_or_else(|| AppError::Config("配置文件没有父目录".to_owned()))?
|
||||
.to_path_buf();
|
||||
let revision = revision(&initial)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(ConfigServiceInner {
|
||||
store,
|
||||
base,
|
||||
startup: initial.clone(),
|
||||
command_line_overrides,
|
||||
state: Mutex::new(ConfigState {
|
||||
revision,
|
||||
config: initial,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> ConfigSnapshot {
|
||||
let state = self
|
||||
.inner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
self.snapshot_from(&state)
|
||||
}
|
||||
|
||||
pub(crate) fn update(
|
||||
&self,
|
||||
request: ConfigUpdateRequest,
|
||||
) -> Result<ConfigSnapshot, ConfigServiceError> {
|
||||
AppConfig::resolve(request.config.clone(), &self.inner.base)
|
||||
.map_err(|error| ConfigServiceError::Validation(error.to_string()))?;
|
||||
let candidate_revision = revision(&request.config)
|
||||
.map_err(|error| ConfigServiceError::Persistence(error.to_string()))?;
|
||||
let mut state = self
|
||||
.inner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if request.revision != state.revision {
|
||||
return Err(ConfigServiceError::Conflict);
|
||||
}
|
||||
self.inner
|
||||
.store
|
||||
.save(&request.config)
|
||||
.map_err(|error| ConfigServiceError::Persistence(error.to_string()))?;
|
||||
state.config = request.config;
|
||||
state.revision = candidate_revision;
|
||||
Ok(self.snapshot_from(&state))
|
||||
}
|
||||
|
||||
fn snapshot_from(&self, state: &ConfigState) -> ConfigSnapshot {
|
||||
ConfigSnapshot {
|
||||
revision: state.revision.clone(),
|
||||
restart_required: state.config != self.inner.startup,
|
||||
source: self.inner.store.path().to_string_lossy().into_owned(),
|
||||
command_line_overrides: self.inner.command_line_overrides.clone(),
|
||||
config: state.config.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn revision(config: &AppConfigDto) -> Result<String, AppError> {
|
||||
let canonical = toml::to_string(config)?;
|
||||
Ok(blake3::hash(canonical.as_bytes()).to_hex().to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::config::TomlConfigStore;
|
||||
|
||||
fn service(directory: &TempDir) -> ConfigService {
|
||||
ConfigService::new(
|
||||
Arc::new(TomlConfigStore::new(directory.path().join("service.toml"))),
|
||||
AppConfigDto::default(),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_update_is_persisted_and_requires_restart() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let service = service(&directory);
|
||||
let current = service.snapshot();
|
||||
let mut config = current.config;
|
||||
config.dht.port = 22_313;
|
||||
let updated = service
|
||||
.update(ConfigUpdateRequest {
|
||||
revision: current.revision,
|
||||
config: config.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(updated.restart_required);
|
||||
assert_eq!(updated.config, config);
|
||||
let stored = TomlConfigStore::new(directory.path().join("service.toml"))
|
||||
.load()
|
||||
.unwrap();
|
||||
assert_eq!(stored, Some(config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_revision_cannot_overwrite_a_newer_update() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let service = service(&directory);
|
||||
let original = service.snapshot();
|
||||
let mut first = original.config.clone();
|
||||
first.dht.port = 22_313;
|
||||
service
|
||||
.update(ConfigUpdateRequest {
|
||||
revision: original.revision.clone(),
|
||||
config: first.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
let mut stale = original.config;
|
||||
stale.dht.port = 32_313;
|
||||
assert!(matches!(
|
||||
service.update(ConfigUpdateRequest {
|
||||
revision: original.revision,
|
||||
config: stale,
|
||||
}),
|
||||
Err(ConfigServiceError::Conflict)
|
||||
));
|
||||
assert_eq!(service.snapshot().config, first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_update_does_not_change_memory_or_disk() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let service = service(&directory);
|
||||
let current = service.snapshot();
|
||||
let mut invalid = current.config.clone();
|
||||
invalid.persistence_queue_capacity = 0;
|
||||
assert!(matches!(
|
||||
service.update(ConfigUpdateRequest {
|
||||
revision: current.revision,
|
||||
config: invalid,
|
||||
}),
|
||||
Err(ConfigServiceError::Validation(_))
|
||||
));
|
||||
assert_eq!(service.snapshot().config, current.config);
|
||||
assert!(!directory.path().join("service.toml").exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// 负责从持久化介质读取并原子保存用户配置 DTO
|
||||
|
||||
use std::{
|
||||
fs::{self, OpenOptions},
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
use crate::error::AppError;
|
||||
|
||||
use super::AppConfigDto;
|
||||
|
||||
static TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub(crate) trait ConfigStore: Send + Sync {
|
||||
fn load(&self) -> Result<Option<AppConfigDto>, AppError>;
|
||||
fn save(&self, config: &AppConfigDto) -> Result<(), AppError>;
|
||||
fn path(&self) -> &Path;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct TomlConfigStore {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl TomlConfigStore {
|
||||
pub(crate) fn new(path: PathBuf) -> Self {
|
||||
Self { path }
|
||||
}
|
||||
}
|
||||
|
||||
impl ConfigStore for TomlConfigStore {
|
||||
fn load(&self) -> Result<Option<AppConfigDto>, AppError> {
|
||||
if !self.path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let contents = fs::read_to_string(&self.path)?;
|
||||
toml::from_str(&contents).map(Some).map_err(AppError::from)
|
||||
}
|
||||
|
||||
fn save(&self, config: &AppConfigDto) -> Result<(), AppError> {
|
||||
let contents = toml::to_string_pretty(config)?;
|
||||
let parent = self
|
||||
.path
|
||||
.parent()
|
||||
.ok_or_else(|| AppError::Config("配置文件没有父目录".to_owned()))?;
|
||||
fs::create_dir_all(parent)?;
|
||||
let temporary = temporary_path(&self.path);
|
||||
let result = (|| {
|
||||
let mut file = OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(&temporary)?;
|
||||
file.write_all(contents.as_bytes())?;
|
||||
file.sync_all()?;
|
||||
drop(file);
|
||||
replace_file(&temporary, &self.path)?;
|
||||
sync_parent(parent)?;
|
||||
Ok::<(), std::io::Error>(())
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&temporary);
|
||||
}
|
||||
result.map_err(AppError::from)
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
fn temporary_path(destination: &Path) -> PathBuf {
|
||||
let sequence = TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let file_name = destination
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("dht-search.toml");
|
||||
destination.with_file_name(format!(
|
||||
".{file_name}.tmp-{}-{sequence}",
|
||||
std::process::id()
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> {
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW,
|
||||
};
|
||||
|
||||
let source: Vec<u16> = source.as_os_str().encode_wide().chain(Some(0)).collect();
|
||||
let destination: Vec<u16> = destination
|
||||
.as_os_str()
|
||||
.encode_wide()
|
||||
.chain(Some(0))
|
||||
.collect();
|
||||
if unsafe {
|
||||
MoveFileExW(
|
||||
source.as_ptr(),
|
||||
destination.as_ptr(),
|
||||
MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
Err(std::io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> {
|
||||
fs::rename(source, destination)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn sync_parent(parent: &Path) -> std::io::Result<()> {
|
||||
std::fs::File::open(parent)?.sync_all()
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn sync_parent(_parent: &Path) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn save_replaces_complete_toml_and_leaves_no_temporary_file() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let path = directory.path().join("service.toml");
|
||||
fs::write(&path, "data_dir = 'old'").unwrap();
|
||||
let store = TomlConfigStore::new(path.clone());
|
||||
let expected = AppConfigDto {
|
||||
data_dir: PathBuf::from("new-data"),
|
||||
..AppConfigDto::default()
|
||||
};
|
||||
store.save(&expected).unwrap();
|
||||
assert_eq!(store.load().unwrap(), Some(expected));
|
||||
assert_eq!(
|
||||
fs::read_dir(directory.path())
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| entry.file_name().to_string_lossy().contains(".tmp-"))
|
||||
.count(),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// 负责将 DHT 采集结果转换为不依赖传输层的领域输入
|
||||
|
||||
use crate::domain::{MetadataCandidate, TorrentFile};
|
||||
use dht_crawler::TorrentInfo;
|
||||
|
||||
pub(crate) fn metadata_candidate(info: TorrentInfo) -> MetadataCandidate {
|
||||
MetadataCandidate {
|
||||
info_hash: info.info_hash,
|
||||
name: info.name,
|
||||
total_size: info.total_size,
|
||||
files: info
|
||||
.files
|
||||
.into_iter()
|
||||
.map(|file| TorrentFile {
|
||||
path: file.path,
|
||||
size: file.size,
|
||||
})
|
||||
.collect(),
|
||||
piece_length: info.piece_length,
|
||||
source_peers: info.peers,
|
||||
timestamp: info.timestamp,
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
// 负责组合 DHT 发现 Metadata 下载和持久化提交管线
|
||||
|
||||
pub(crate) mod mapper;
|
||||
pub(crate) mod pipeline;
|
||||
@@ -10,13 +10,14 @@ use std::{
|
||||
thread::{self, JoinHandle},
|
||||
};
|
||||
|
||||
use dht_crawler::TorrentInfo;
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
domain::{InfoHash, MetadataLimits, MetadataRejectionReason, RejectedMetadata, TorrentRecord},
|
||||
storage::{StorageError, TorrentRepository, UpsertOutcome},
|
||||
};
|
||||
use dht_crawler::TorrentInfo;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
use crate::crawler::mapper::metadata_candidate;
|
||||
use crate::disk_guard::DiskGuard;
|
||||
use crate::error::AppError;
|
||||
|
||||
@@ -145,25 +146,26 @@ impl PersistenceIngress {
|
||||
};
|
||||
let rejected_info_hash = InfoHash::from_str(&torrent.info_hash).ok();
|
||||
let rejected_at = torrent.timestamp;
|
||||
let record = match TorrentRecord::try_from_with_limits(torrent, self.limits) {
|
||||
Ok(record) => record,
|
||||
Err(error) => {
|
||||
self.stats.invalid.fetch_add(1, Ordering::Relaxed);
|
||||
let reason = error.rejection_reason();
|
||||
self.stats.filtered[reason.index()].fetch_add(1, Ordering::Relaxed);
|
||||
tracing::warn!(%error, ?reason, "拒绝无效 Metadata");
|
||||
if let Some(info_hash) = rejected_info_hash {
|
||||
let rejection = RejectedMetadata::new(
|
||||
info_hash,
|
||||
reason,
|
||||
self.limits.rule_id(),
|
||||
rejected_at,
|
||||
);
|
||||
self.try_send(PersistenceItem::Rejection(rejection), false);
|
||||
let record =
|
||||
match TorrentRecord::try_from_with_limits(metadata_candidate(torrent), self.limits) {
|
||||
Ok(record) => record,
|
||||
Err(error) => {
|
||||
self.stats.invalid.fetch_add(1, Ordering::Relaxed);
|
||||
let reason = error.rejection_reason();
|
||||
self.stats.filtered[reason.index()].fetch_add(1, Ordering::Relaxed);
|
||||
tracing::warn!(%error, ?reason, "拒绝无效 Metadata");
|
||||
if let Some(info_hash) = rejected_info_hash {
|
||||
let rejection = RejectedMetadata::new(
|
||||
info_hash,
|
||||
reason,
|
||||
self.limits.rule_id(),
|
||||
rejected_at,
|
||||
);
|
||||
self.try_send(PersistenceItem::Rejection(rejection), false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
};
|
||||
self.try_send(PersistenceItem::Record(record), true)
|
||||
}
|
||||
|
||||
@@ -234,14 +236,14 @@ impl MetadataFilterSnapshot {
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
|
||||
use dht_crawler::FileInfo;
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
domain::{ContentGroup, InfoHash, VerificationResult},
|
||||
storage::{
|
||||
ContentGroupTask, ContentVariants, UpsertOutcome, VerificationEnqueueOutcome,
|
||||
VerificationPriority, VerificationRequest,
|
||||
},
|
||||
};
|
||||
use dht_crawler::FileInfo;
|
||||
|
||||
use super::*;
|
||||
use crate::config::DiskGuardConfig;
|
||||
@@ -0,0 +1,166 @@
|
||||
// 负责记录 HTTP 请求并发状态错误分类和固定桶延迟分布
|
||||
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use super::model::HttpDiagnostics;
|
||||
|
||||
const LATENCY_BUCKETS_MICROS: [u64; 11] = [
|
||||
1_000,
|
||||
5_000,
|
||||
10_000,
|
||||
25_000,
|
||||
50_000,
|
||||
100_000,
|
||||
250_000,
|
||||
500_000,
|
||||
1_000_000,
|
||||
5_000_000,
|
||||
u64::MAX,
|
||||
];
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct HttpStats {
|
||||
inner: Arc<HttpStatsInner>,
|
||||
}
|
||||
|
||||
struct HttpStatsInner {
|
||||
active: AtomicU64,
|
||||
requests: AtomicU64,
|
||||
client_errors: AtomicU64,
|
||||
server_errors: AtomicU64,
|
||||
latency_total_micros: AtomicU64,
|
||||
latency_max_micros: AtomicU64,
|
||||
latency_buckets: [AtomicU64; LATENCY_BUCKETS_MICROS.len()],
|
||||
}
|
||||
|
||||
impl Default for HttpStatsInner {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
active: AtomicU64::new(0),
|
||||
requests: AtomicU64::new(0),
|
||||
client_errors: AtomicU64::new(0),
|
||||
server_errors: AtomicU64::new(0),
|
||||
latency_total_micros: AtomicU64::new(0),
|
||||
latency_max_micros: AtomicU64::new(0),
|
||||
latency_buckets: std::array::from_fn(|_| AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct HttpRequestTimer {
|
||||
stats: HttpStats,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
impl HttpStats {
|
||||
pub(crate) fn begin(&self) -> HttpRequestTimer {
|
||||
self.inner.active.fetch_add(1, Ordering::Relaxed);
|
||||
HttpRequestTimer {
|
||||
stats: self.clone(),
|
||||
started: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> HttpDiagnostics {
|
||||
let requests = self.inner.requests.load(Ordering::Relaxed);
|
||||
let total_micros = self.inner.latency_total_micros.load(Ordering::Relaxed);
|
||||
HttpDiagnostics {
|
||||
active_requests: self.inner.active.load(Ordering::Relaxed),
|
||||
requests,
|
||||
client_errors: self.inner.client_errors.load(Ordering::Relaxed),
|
||||
server_errors: self.inner.server_errors.load(Ordering::Relaxed),
|
||||
latency_average_micros: (requests != 0).then(|| total_micros / requests),
|
||||
latency_p95_millis: percentile_millis(&self.inner.latency_buckets, requests, 95),
|
||||
latency_max_millis: micros_to_millis(
|
||||
self.inner.latency_max_micros.load(Ordering::Relaxed),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, status: u16, elapsed_micros: u64) {
|
||||
self.inner.requests.fetch_add(1, Ordering::Relaxed);
|
||||
if (400..500).contains(&status) {
|
||||
self.inner.client_errors.fetch_add(1, Ordering::Relaxed);
|
||||
} else if status >= 500 {
|
||||
self.inner.server_errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let _ = self.inner.latency_total_micros.fetch_update(
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
|value| Some(value.saturating_add(elapsed_micros)),
|
||||
);
|
||||
self.inner
|
||||
.latency_max_micros
|
||||
.fetch_max(elapsed_micros, Ordering::Relaxed);
|
||||
let index = LATENCY_BUCKETS_MICROS
|
||||
.iter()
|
||||
.position(|upper| elapsed_micros <= *upper)
|
||||
.unwrap_or(LATENCY_BUCKETS_MICROS.len() - 1);
|
||||
self.inner.latency_buckets[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl HttpRequestTimer {
|
||||
pub(crate) fn finish(self, status: u16) {
|
||||
let elapsed_micros = self.started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64;
|
||||
self.stats.observe(status, elapsed_micros);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for HttpRequestTimer {
|
||||
fn drop(&mut self) {
|
||||
self.stats.inner.active.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn percentile_millis(
|
||||
buckets: &[AtomicU64; LATENCY_BUCKETS_MICROS.len()],
|
||||
count: u64,
|
||||
percentile: u64,
|
||||
) -> Option<u64> {
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
let rank = count.saturating_mul(percentile).saturating_add(99) / 100;
|
||||
let mut cumulative = 0_u64;
|
||||
for (index, bucket) in buckets.iter().enumerate() {
|
||||
cumulative = cumulative.saturating_add(bucket.load(Ordering::Relaxed));
|
||||
if cumulative >= rank {
|
||||
let upper = LATENCY_BUCKETS_MICROS[index];
|
||||
return (upper != u64::MAX).then(|| micros_to_millis(upper));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn micros_to_millis(micros: u64) -> u64 {
|
||||
micros.saturating_add(999) / 1_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn snapshots_track_active_completed_and_error_requests() {
|
||||
let stats = HttpStats::default();
|
||||
let active = stats.begin();
|
||||
assert_eq!(stats.snapshot().active_requests, 1);
|
||||
active.finish(404);
|
||||
let failed = stats.begin();
|
||||
failed.finish(503);
|
||||
let snapshot = stats.snapshot();
|
||||
assert_eq!(snapshot.active_requests, 0);
|
||||
assert_eq!(snapshot.requests, 2);
|
||||
assert_eq!(snapshot.client_errors, 1);
|
||||
assert_eq!(snapshot.server_errors, 1);
|
||||
assert!(snapshot.latency_average_micros.is_some());
|
||||
assert!(snapshot.latency_p95_millis.is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
// 负责采集运行资源快照并协调有界 SQLite 历史写入和查询
|
||||
|
||||
mod http;
|
||||
mod model;
|
||||
mod process;
|
||||
mod store;
|
||||
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
Arc, Mutex, RwLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
mpsc::{SyncSender, TrySendError, sync_channel},
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, StorageDiagnostics as RocksDiagnostics},
|
||||
};
|
||||
use dht_crawler::DhtRuntimeStats;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::{
|
||||
config::DiagnosticsConfig, crawler::pipeline::PersistenceIngress, disk_guard::DiskGuard,
|
||||
};
|
||||
|
||||
pub(crate) use http::HttpStats;
|
||||
pub(crate) use model::{
|
||||
CurrentDiagnosticsResponse, DiagnosticHistory, DiagnosticSample, HistoryResolution,
|
||||
};
|
||||
use model::{DiagnosticsStatus, RuntimeDiagnostics, SearchDiagnostics, StorageDiagnostics};
|
||||
use store::{DiagnosticStore, DiagnosticStoreError};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DiagnosticSources {
|
||||
pub(crate) repository: Arc<RocksTorrentRepository>,
|
||||
pub(crate) search: SearchEngine,
|
||||
pub(crate) dht: DhtRuntimeStats,
|
||||
pub(crate) persistence: PersistenceIngress,
|
||||
pub(crate) disk_guard: DiskGuard,
|
||||
pub(crate) http: HttpStats,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DiagnosticsHandle {
|
||||
state: Arc<DiagnosticsState>,
|
||||
database: Option<PathBuf>,
|
||||
}
|
||||
|
||||
pub(crate) struct DiagnosticsRuntime {
|
||||
handle: DiagnosticsHandle,
|
||||
cancel: CancellationToken,
|
||||
collector: Option<tokio::task::JoinHandle<()>>,
|
||||
writer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DiagnosticsState {
|
||||
enabled: bool,
|
||||
current: RwLock<Option<DiagnosticSample>>,
|
||||
persisted_samples: AtomicU64,
|
||||
dropped_samples: AtomicU64,
|
||||
skipped_samples: AtomicU64,
|
||||
write_failures: AtomicU64,
|
||||
last_persisted_at: AtomicU64,
|
||||
last_error: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum DiagnosticsError {
|
||||
#[error("运行诊断线程启动失败: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error(transparent)]
|
||||
Store(#[from] DiagnosticStoreError),
|
||||
#[error("运行诊断历史未启用")]
|
||||
Disabled,
|
||||
#[error("运行诊断 writer 异常退出")]
|
||||
WriterPanicked,
|
||||
}
|
||||
|
||||
impl DiagnosticsRuntime {
|
||||
pub(crate) fn disabled() -> Self {
|
||||
Self {
|
||||
handle: DiagnosticsHandle {
|
||||
state: Arc::new(DiagnosticsState::default()),
|
||||
database: None,
|
||||
},
|
||||
cancel: CancellationToken::new(),
|
||||
collector: None,
|
||||
writer: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start(
|
||||
config: DiagnosticsConfig,
|
||||
sources: DiagnosticSources,
|
||||
) -> Result<Self, DiagnosticsError> {
|
||||
let store = DiagnosticStore::open(&config.database)?;
|
||||
let state = Arc::new(DiagnosticsState {
|
||||
enabled: true,
|
||||
..DiagnosticsState::default()
|
||||
});
|
||||
let (sender, receiver) = sync_channel(config.queue_capacity);
|
||||
let writer_state = state.clone();
|
||||
let writer_config = config.clone();
|
||||
let writer_disk_guard = sources.disk_guard.clone();
|
||||
let writer = std::thread::Builder::new()
|
||||
.name("diagnostics-sqlite".to_owned())
|
||||
.spawn(move || {
|
||||
let mut store = store;
|
||||
while let Ok(sample) = receiver.recv() {
|
||||
let Some(_permit) = writer_disk_guard.begin_new_write() else {
|
||||
writer_state.skipped_samples.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
};
|
||||
match store.record(&sample, &writer_config) {
|
||||
Ok(()) => {
|
||||
writer_state
|
||||
.persisted_samples
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
writer_state
|
||||
.last_persisted_at
|
||||
.store(sample.captured_at, Ordering::Relaxed);
|
||||
}
|
||||
Err(error) => {
|
||||
writer_state.write_failures.fetch_add(1, Ordering::Relaxed);
|
||||
set_last_error(&writer_state, error.to_string());
|
||||
tracing::warn!(%error, "运行诊断采样写入失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
})?;
|
||||
let cancel = CancellationToken::new();
|
||||
let collector = tokio::spawn(collect_loop(
|
||||
sources,
|
||||
sender,
|
||||
state.clone(),
|
||||
config.sample_interval_secs,
|
||||
cancel.clone(),
|
||||
));
|
||||
Ok(Self {
|
||||
handle: DiagnosticsHandle {
|
||||
state,
|
||||
database: Some(config.database),
|
||||
},
|
||||
cancel,
|
||||
collector: Some(collector),
|
||||
writer: Some(writer),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn handle(&self) -> DiagnosticsHandle {
|
||||
self.handle.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn request_shutdown(&self) {
|
||||
self.cancel.cancel();
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(mut self) -> Result<(), DiagnosticsError> {
|
||||
self.cancel.cancel();
|
||||
if let Some(collector) = self.collector.take() {
|
||||
let _ = collector.await;
|
||||
}
|
||||
if let Some(writer) = self.writer.take() {
|
||||
tokio::task::spawn_blocking(move || writer.join())
|
||||
.await
|
||||
.map_err(|_| DiagnosticsError::WriterPanicked)?
|
||||
.map_err(|_| DiagnosticsError::WriterPanicked)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl DiagnosticsHandle {
|
||||
pub(crate) fn current(&self) -> CurrentDiagnosticsResponse {
|
||||
let last_persisted_at = self.state.last_persisted_at.load(Ordering::Relaxed);
|
||||
CurrentDiagnosticsResponse {
|
||||
status: DiagnosticsStatus {
|
||||
enabled: self.state.enabled,
|
||||
persisted_samples: self.state.persisted_samples.load(Ordering::Relaxed),
|
||||
dropped_samples: self.state.dropped_samples.load(Ordering::Relaxed),
|
||||
skipped_samples: self.state.skipped_samples.load(Ordering::Relaxed),
|
||||
write_failures: self.state.write_failures.load(Ordering::Relaxed),
|
||||
last_persisted_at: (last_persisted_at != 0).then_some(last_persisted_at),
|
||||
last_error: self
|
||||
.state
|
||||
.last_error
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone(),
|
||||
},
|
||||
sample: self
|
||||
.state
|
||||
.current
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn history(
|
||||
&self,
|
||||
resolution: HistoryResolution,
|
||||
from: u64,
|
||||
to: u64,
|
||||
) -> Result<DiagnosticHistory, DiagnosticsError> {
|
||||
let database = self.database.as_ref().ok_or(DiagnosticsError::Disabled)?;
|
||||
DiagnosticStore::history(database, resolution, from, to).map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_loop(
|
||||
sources: DiagnosticSources,
|
||||
sender: SyncSender<DiagnosticSample>,
|
||||
state: Arc<DiagnosticsState>,
|
||||
interval_secs: u64,
|
||||
cancel: CancellationToken,
|
||||
) {
|
||||
let session_started_at = unix_timestamp();
|
||||
let mut ticker = tokio::time::interval(Duration::from_secs(interval_secs));
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => break,
|
||||
_ = ticker.tick() => {
|
||||
let sources = sources.clone();
|
||||
let sample = match tokio::task::spawn_blocking(move || {
|
||||
collect_sample(&sources, session_started_at)
|
||||
}).await {
|
||||
Ok(sample) => sample,
|
||||
Err(error) => {
|
||||
state.dropped_samples.fetch_add(1, Ordering::Relaxed);
|
||||
set_last_error(&state, error.to_string());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
*state
|
||||
.current
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(sample.clone());
|
||||
match sender.try_send(sample) {
|
||||
Ok(()) => {}
|
||||
Err(TrySendError::Full(_)) => {
|
||||
state.dropped_samples.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(TrySendError::Disconnected(_)) => {
|
||||
state.dropped_samples.fetch_add(1, Ordering::Relaxed);
|
||||
set_last_error(&state, "运行诊断 writer 已停止".to_owned());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_sample(sources: &DiagnosticSources, session_started_at: u64) -> DiagnosticSample {
|
||||
let dht = sources.dht.snapshot();
|
||||
let observability = sources.dht.observability_snapshot();
|
||||
let persistence = sources.persistence.snapshot();
|
||||
let disk = sources.disk_guard.snapshot();
|
||||
let storage = sources.repository.diagnostics().unwrap_or_else(|error| {
|
||||
tracing::warn!(%error, "读取 RocksDB 诊断属性失败");
|
||||
RocksDiagnostics::default()
|
||||
});
|
||||
let search = sources.search.diagnostics();
|
||||
DiagnosticSample {
|
||||
captured_at: unix_timestamp(),
|
||||
session_started_at,
|
||||
process: process::snapshot(),
|
||||
storage: StorageDiagnostics {
|
||||
block_cache_bytes: storage.block_cache_bytes,
|
||||
memtable_bytes: storage.memtable_bytes,
|
||||
pending_compaction_bytes: storage.pending_compaction_bytes,
|
||||
live_sst_bytes: storage.live_sst_bytes,
|
||||
running_compactions: storage.running_compactions,
|
||||
estimated_keys: storage.estimated_keys,
|
||||
},
|
||||
search: SearchDiagnostics {
|
||||
documents: search.documents,
|
||||
writer_memory_budget_bytes: search.writer_memory_budget_bytes,
|
||||
commits: search.commits,
|
||||
commit_failures: search.commit_failures,
|
||||
last_commit_at: search.last_commit_at,
|
||||
last_commit_duration_millis: search.last_commit_duration_millis,
|
||||
last_commit_documents: search.last_commit_documents,
|
||||
},
|
||||
http: sources.http.snapshot(),
|
||||
runtime: RuntimeDiagnostics {
|
||||
nodes: saturating_u64(dht.node_pool_size),
|
||||
udp_tx_packets: observability.udp_tx_packets,
|
||||
metadata_in_flight: saturating_u64(dht.metadata_in_flight),
|
||||
metadata_succeeded: dht.metadata_peer_succeeded,
|
||||
metadata_failed: dht.metadata_peer_failed,
|
||||
sample_queue_depth: saturating_u64(dht.sample_candidate_queue_depth),
|
||||
sample_queue_capacity: saturating_u64(dht.sample_candidate_queue_capacity),
|
||||
persistence_queue_depth: saturating_u64(persistence.queue_depth),
|
||||
indexed_documents: search.documents,
|
||||
disk_available_bytes: disk.available_bytes,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn set_last_error(state: &DiagnosticsState, message: String) {
|
||||
*state
|
||||
.last_error
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(message);
|
||||
}
|
||||
|
||||
fn saturating_u64(value: usize) -> u64 {
|
||||
value.min(u64::MAX as usize) as u64
|
||||
}
|
||||
|
||||
fn unix_timestamp() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
config::DiskGuardConfig, crawler::pipeline::PersistencePipeline, disk_guard::DiskGuard,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_persists_a_queryable_snapshot_and_stops_cleanly() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let repository =
|
||||
Arc::new(RocksTorrentRepository::open(directory.path().join("rocksdb")).unwrap());
|
||||
let repository_trait: Arc<dyn TorrentRepository> = repository.clone();
|
||||
let disk_guard = DiskGuard::new(&DiskGuardConfig {
|
||||
enabled: false,
|
||||
..DiskGuardConfig::default()
|
||||
});
|
||||
let persistence = PersistencePipeline::start(
|
||||
repository_trait,
|
||||
4,
|
||||
crate::domain::MetadataLimits::default(),
|
||||
disk_guard.clone(),
|
||||
);
|
||||
let config = DiagnosticsConfig {
|
||||
database: directory.path().join("diagnostics.sqlite3"),
|
||||
sample_interval_secs: 1,
|
||||
..DiagnosticsConfig::default()
|
||||
};
|
||||
let runtime = DiagnosticsRuntime::start(
|
||||
config,
|
||||
DiagnosticSources {
|
||||
repository,
|
||||
search: SearchEngine::open(directory.path().join("tantivy")).unwrap(),
|
||||
dht: DhtRuntimeStats::default(),
|
||||
persistence: persistence.ingress.clone(),
|
||||
disk_guard,
|
||||
http: HttpStats::default(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let handle = runtime.handle();
|
||||
for _ in 0..40 {
|
||||
if handle.current().status.persisted_samples > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
let current = handle.current();
|
||||
assert!(current.status.enabled);
|
||||
assert_eq!(current.status.write_failures, 0);
|
||||
assert!(current.sample.is_some());
|
||||
let now = unix_timestamp();
|
||||
let history = handle
|
||||
.history(HistoryResolution::Raw, now.saturating_sub(5), now)
|
||||
.unwrap();
|
||||
assert!(!history.samples.is_empty());
|
||||
|
||||
runtime.shutdown().await.unwrap();
|
||||
persistence.close_and_join().await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// 负责定义运行诊断采样状态和历史查询 DTO
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct DiagnosticSample {
|
||||
pub(crate) captured_at: u64,
|
||||
pub(crate) session_started_at: u64,
|
||||
pub(crate) process: ProcessDiagnostics,
|
||||
pub(crate) storage: StorageDiagnostics,
|
||||
pub(crate) search: SearchDiagnostics,
|
||||
#[serde(default)]
|
||||
pub(crate) http: HttpDiagnostics,
|
||||
pub(crate) runtime: RuntimeDiagnostics,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct ProcessDiagnostics {
|
||||
pub(crate) resident_memory_bytes: Option<u64>,
|
||||
pub(crate) private_memory_bytes: Option<u64>,
|
||||
pub(crate) cpu_time_millis: Option<u64>,
|
||||
pub(crate) thread_count: Option<u64>,
|
||||
pub(crate) handle_count: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct StorageDiagnostics {
|
||||
pub(crate) block_cache_bytes: Option<u64>,
|
||||
pub(crate) memtable_bytes: Option<u64>,
|
||||
pub(crate) pending_compaction_bytes: Option<u64>,
|
||||
pub(crate) live_sst_bytes: Option<u64>,
|
||||
pub(crate) running_compactions: Option<u64>,
|
||||
pub(crate) estimated_keys: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct SearchDiagnostics {
|
||||
pub(crate) documents: u64,
|
||||
pub(crate) writer_memory_budget_bytes: u64,
|
||||
pub(crate) commits: u64,
|
||||
pub(crate) commit_failures: u64,
|
||||
pub(crate) last_commit_at: Option<u64>,
|
||||
pub(crate) last_commit_duration_millis: u64,
|
||||
pub(crate) last_commit_documents: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct HttpDiagnostics {
|
||||
pub(crate) active_requests: u64,
|
||||
pub(crate) requests: u64,
|
||||
pub(crate) client_errors: u64,
|
||||
pub(crate) server_errors: u64,
|
||||
pub(crate) latency_average_micros: Option<u64>,
|
||||
pub(crate) latency_p95_millis: Option<u64>,
|
||||
pub(crate) latency_max_millis: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) struct RuntimeDiagnostics {
|
||||
pub(crate) nodes: u64,
|
||||
pub(crate) udp_tx_packets: u64,
|
||||
pub(crate) metadata_in_flight: u64,
|
||||
pub(crate) metadata_succeeded: u64,
|
||||
pub(crate) metadata_failed: u64,
|
||||
pub(crate) sample_queue_depth: u64,
|
||||
pub(crate) sample_queue_capacity: u64,
|
||||
pub(crate) persistence_queue_depth: u64,
|
||||
pub(crate) indexed_documents: u64,
|
||||
pub(crate) disk_available_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub(crate) struct DiagnosticsStatus {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) persisted_samples: u64,
|
||||
pub(crate) dropped_samples: u64,
|
||||
pub(crate) skipped_samples: u64,
|
||||
pub(crate) write_failures: u64,
|
||||
pub(crate) last_persisted_at: Option<u64>,
|
||||
pub(crate) last_error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct CurrentDiagnosticsResponse {
|
||||
pub(crate) status: DiagnosticsStatus,
|
||||
pub(crate) sample: Option<DiagnosticSample>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct DiagnosticHistory {
|
||||
pub(crate) resolution: &'static str,
|
||||
pub(crate) from: u64,
|
||||
pub(crate) to: u64,
|
||||
pub(crate) samples: Vec<DiagnosticSample>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HistoryResolution {
|
||||
Raw,
|
||||
Minute,
|
||||
}
|
||||
|
||||
impl HistoryResolution {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Raw => "raw",
|
||||
Self::Minute => "minute",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn database_value(self) -> i64 {
|
||||
match self {
|
||||
Self::Raw => 0,
|
||||
Self::Minute => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// 负责采集当前进程内存 CPU 线程和句柄资源快照
|
||||
|
||||
use super::model::ProcessDiagnostics;
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn snapshot() -> ProcessDiagnostics {
|
||||
use std::mem::size_of;
|
||||
|
||||
use windows_sys::Win32::{
|
||||
Foundation::{CloseHandle, FILETIME, INVALID_HANDLE_VALUE},
|
||||
System::{
|
||||
Diagnostics::ToolHelp::{
|
||||
CreateToolhelp32Snapshot, TH32CS_SNAPTHREAD, THREADENTRY32, Thread32First,
|
||||
Thread32Next,
|
||||
},
|
||||
ProcessStatus::{
|
||||
K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX,
|
||||
},
|
||||
Threading::{
|
||||
GetCurrentProcess, GetCurrentProcessId, GetProcessHandleCount, GetProcessTimes,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
let process = unsafe { GetCurrentProcess() };
|
||||
let mut memory = PROCESS_MEMORY_COUNTERS_EX {
|
||||
cb: size_of::<PROCESS_MEMORY_COUNTERS_EX>() as u32,
|
||||
..PROCESS_MEMORY_COUNTERS_EX::default()
|
||||
};
|
||||
let memory_ok = unsafe {
|
||||
K32GetProcessMemoryInfo(
|
||||
process,
|
||||
(&raw mut memory).cast::<PROCESS_MEMORY_COUNTERS>(),
|
||||
memory.cb,
|
||||
) != 0
|
||||
};
|
||||
|
||||
let mut handles = 0_u32;
|
||||
let handles_ok = unsafe { GetProcessHandleCount(process, &raw mut handles) != 0 };
|
||||
|
||||
let mut creation = FILETIME::default();
|
||||
let mut exit = FILETIME::default();
|
||||
let mut kernel = FILETIME::default();
|
||||
let mut user = FILETIME::default();
|
||||
let cpu_ok = unsafe {
|
||||
GetProcessTimes(
|
||||
process,
|
||||
&raw mut creation,
|
||||
&raw mut exit,
|
||||
&raw mut kernel,
|
||||
&raw mut user,
|
||||
) != 0
|
||||
};
|
||||
|
||||
let result = ProcessDiagnostics {
|
||||
resident_memory_bytes: memory_ok.then_some(memory.WorkingSetSize as u64),
|
||||
private_memory_bytes: memory_ok.then_some(memory.PrivateUsage as u64),
|
||||
cpu_time_millis: cpu_ok
|
||||
.then(|| filetime_ticks(kernel).saturating_add(filetime_ticks(user)) / 10_000),
|
||||
thread_count: windows_thread_count(),
|
||||
handle_count: handles_ok.then_some(u64::from(handles)),
|
||||
};
|
||||
|
||||
fn filetime_ticks(value: FILETIME) -> u64 {
|
||||
(u64::from(value.dwHighDateTime) << 32) | u64::from(value.dwLowDateTime)
|
||||
}
|
||||
|
||||
fn windows_thread_count() -> Option<u64> {
|
||||
let snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0) };
|
||||
if snapshot == INVALID_HANDLE_VALUE {
|
||||
return None;
|
||||
}
|
||||
let process_id = unsafe { GetCurrentProcessId() };
|
||||
let mut entry = THREADENTRY32 {
|
||||
dwSize: size_of::<THREADENTRY32>() as u32,
|
||||
..THREADENTRY32::default()
|
||||
};
|
||||
let mut count = 0_u64;
|
||||
let mut has_entry = unsafe { Thread32First(snapshot, &raw mut entry) != 0 };
|
||||
while has_entry {
|
||||
if entry.th32OwnerProcessID == process_id {
|
||||
count = count.saturating_add(1);
|
||||
}
|
||||
has_entry = unsafe { Thread32Next(snapshot, &raw mut entry) != 0 };
|
||||
}
|
||||
unsafe {
|
||||
CloseHandle(snapshot);
|
||||
}
|
||||
Some(count)
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn snapshot() -> ProcessDiagnostics {
|
||||
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
|
||||
ProcessDiagnostics {
|
||||
resident_memory_bytes: status_kib(&status, "VmRSS:"),
|
||||
private_memory_bytes: status_kib(&status, "RssAnon:")
|
||||
.or_else(|| status_kib(&status, "VmData:")),
|
||||
cpu_time_millis: linux_cpu_time_millis(),
|
||||
thread_count: status_value(&status, "Threads:"),
|
||||
handle_count: std::fs::read_dir("/proc/self/fd")
|
||||
.ok()
|
||||
.map(|entries| entries.count().min(u64::MAX as usize) as u64),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn status_kib(status: &str, key: &str) -> Option<u64> {
|
||||
status_value(status, key).map(|value| value.saturating_mul(1_024))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn status_value(status: &str, key: &str) -> Option<u64> {
|
||||
status
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix(key))?
|
||||
.split_whitespace()
|
||||
.next()?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn linux_cpu_time_millis() -> Option<u64> {
|
||||
let stat = std::fs::read_to_string("/proc/self/stat").ok()?;
|
||||
let fields = stat.get(stat.rfind(')')?.saturating_add(2)..)?;
|
||||
let mut fields = fields.split_whitespace();
|
||||
let user_ticks: u64 = fields.nth(11)?.parse().ok()?;
|
||||
let system_ticks: u64 = fields.next()?.parse().ok()?;
|
||||
let ticks_per_second = unsafe { libc::sysconf(libc::_SC_CLK_TCK) };
|
||||
if ticks_per_second <= 0 {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
user_ticks
|
||||
.saturating_add(system_ticks)
|
||||
.saturating_mul(1_000)
|
||||
/ ticks_per_second as u64,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
pub(crate) fn snapshot() -> ProcessDiagnostics {
|
||||
ProcessDiagnostics::default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn current_process_reports_available_platform_resources() {
|
||||
let snapshot = super::snapshot();
|
||||
#[cfg(any(windows, target_os = "linux"))]
|
||||
{
|
||||
assert!(
|
||||
snapshot
|
||||
.resident_memory_bytes
|
||||
.is_some_and(|value| value > 0)
|
||||
);
|
||||
assert!(snapshot.cpu_time_millis.is_some());
|
||||
assert!(snapshot.thread_count.is_some_and(|value| value > 0));
|
||||
assert!(snapshot.handle_count.is_some_and(|value| value > 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
// 负责使用独立 SQLite 数据库持久化和查询有界诊断历史
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use rusqlite::{Connection, OpenFlags, params};
|
||||
|
||||
use crate::config::DiagnosticsConfig;
|
||||
|
||||
use super::model::{DiagnosticHistory, DiagnosticSample, HistoryResolution};
|
||||
|
||||
const DATABASE_VERSION: i64 = 1;
|
||||
const MAX_HISTORY_SAMPLES: usize = 50_000;
|
||||
|
||||
pub(crate) struct DiagnosticStore {
|
||||
connection: Connection,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum DiagnosticStoreError {
|
||||
#[error("无法准备诊断数据库目录: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("SQLite 诊断数据库操作失败: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
#[error("诊断采样序列化失败: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("诊断数据库格式版本不兼容")]
|
||||
IncompatibleVersion,
|
||||
}
|
||||
|
||||
impl DiagnosticStore {
|
||||
pub(crate) fn open(path: &Path) -> Result<Self, DiagnosticStoreError> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let connection = Connection::open(path)?;
|
||||
configure(&connection)?;
|
||||
initialize(&connection)?;
|
||||
Ok(Self { connection })
|
||||
}
|
||||
|
||||
pub(crate) fn record(
|
||||
&mut self,
|
||||
sample: &DiagnosticSample,
|
||||
config: &DiagnosticsConfig,
|
||||
) -> Result<(), DiagnosticStoreError> {
|
||||
let payload = serde_json::to_string(sample)?;
|
||||
let transaction = self.connection.transaction()?;
|
||||
transaction.execute(
|
||||
"INSERT OR REPLACE INTO diagnostic_samples (kind, captured_at, payload) VALUES (0, ?1, ?2)",
|
||||
params![as_i64(sample.captured_at), payload],
|
||||
)?;
|
||||
let minute = sample.captured_at / 60 * 60;
|
||||
transaction.execute(
|
||||
"INSERT OR REPLACE INTO diagnostic_samples (kind, captured_at, payload) VALUES (1, ?1, ?2)",
|
||||
params![as_i64(minute), payload],
|
||||
)?;
|
||||
let raw_cutoff = sample
|
||||
.captured_at
|
||||
.saturating_sub(config.raw_retention_hours.saturating_mul(3_600));
|
||||
let minute_cutoff = sample
|
||||
.captured_at
|
||||
.saturating_sub(config.minute_retention_days.saturating_mul(86_400));
|
||||
transaction.execute(
|
||||
"DELETE FROM diagnostic_samples WHERE kind = 0 AND captured_at < ?1",
|
||||
[as_i64(raw_cutoff)],
|
||||
)?;
|
||||
transaction.execute(
|
||||
"DELETE FROM diagnostic_samples WHERE kind = 1 AND captured_at < ?1",
|
||||
[as_i64(minute_cutoff)],
|
||||
)?;
|
||||
transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn history(
|
||||
path: &Path,
|
||||
resolution: HistoryResolution,
|
||||
from: u64,
|
||||
to: u64,
|
||||
) -> Result<DiagnosticHistory, DiagnosticStoreError> {
|
||||
let connection = Connection::open_with_flags(
|
||||
path,
|
||||
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
|
||||
)?;
|
||||
let mut statement = connection.prepare(
|
||||
"SELECT payload FROM diagnostic_samples
|
||||
WHERE kind = ?1 AND captured_at >= ?2 AND captured_at <= ?3
|
||||
ORDER BY captured_at ASC LIMIT ?4",
|
||||
)?;
|
||||
let rows = statement.query_map(
|
||||
params![
|
||||
resolution.database_value(),
|
||||
as_i64(from),
|
||||
as_i64(to),
|
||||
MAX_HISTORY_SAMPLES as i64
|
||||
],
|
||||
|row| row.get::<_, String>(0),
|
||||
)?;
|
||||
let mut samples = Vec::new();
|
||||
for payload in rows {
|
||||
samples.push(serde_json::from_str(&payload?)?);
|
||||
}
|
||||
Ok(DiagnosticHistory {
|
||||
resolution: resolution.as_str(),
|
||||
from,
|
||||
to,
|
||||
samples,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn count(&self, resolution: HistoryResolution) -> Result<u64, DiagnosticStoreError> {
|
||||
let count: i64 = self
|
||||
.connection
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM diagnostic_samples WHERE kind = ?1",
|
||||
[resolution.database_value()],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map_err(DiagnosticStoreError::from)?;
|
||||
Ok(count.max(0) as u64)
|
||||
}
|
||||
}
|
||||
|
||||
fn configure(connection: &Connection) -> Result<(), rusqlite::Error> {
|
||||
connection.pragma_update(None, "journal_mode", "WAL")?;
|
||||
connection.pragma_update(None, "synchronous", "NORMAL")?;
|
||||
connection.busy_timeout(std::time::Duration::from_secs(5))
|
||||
}
|
||||
|
||||
fn initialize(connection: &Connection) -> Result<(), DiagnosticStoreError> {
|
||||
let version: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?;
|
||||
if version != 0 && version != DATABASE_VERSION {
|
||||
return Err(DiagnosticStoreError::IncompatibleVersion);
|
||||
}
|
||||
connection.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS diagnostic_samples (
|
||||
kind INTEGER NOT NULL,
|
||||
captured_at INTEGER NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
PRIMARY KEY (kind, captured_at)
|
||||
) WITHOUT ROWID;",
|
||||
)?;
|
||||
connection.pragma_update(None, "user_version", DATABASE_VERSION)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn as_i64(value: u64) -> i64 {
|
||||
value.min(i64::MAX as u64) as i64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::diagnostics::model::{
|
||||
HttpDiagnostics, ProcessDiagnostics, RuntimeDiagnostics, SearchDiagnostics,
|
||||
StorageDiagnostics,
|
||||
};
|
||||
|
||||
fn sample(captured_at: u64) -> DiagnosticSample {
|
||||
DiagnosticSample {
|
||||
captured_at,
|
||||
session_started_at: 1,
|
||||
process: ProcessDiagnostics::default(),
|
||||
storage: StorageDiagnostics::default(),
|
||||
search: SearchDiagnostics::default(),
|
||||
http: HttpDiagnostics::default(),
|
||||
runtime: RuntimeDiagnostics::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn samples_survive_reopen_and_minute_rows_are_compacted() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let path = directory.path().join("diagnostics.sqlite3");
|
||||
let config = DiagnosticsConfig {
|
||||
raw_retention_hours: 1,
|
||||
minute_retention_days: 1,
|
||||
..DiagnosticsConfig::default()
|
||||
};
|
||||
let mut store = DiagnosticStore::open(&path).unwrap();
|
||||
store.record(&sample(3_600), &config).unwrap();
|
||||
store.record(&sample(3_610), &config).unwrap();
|
||||
assert_eq!(store.count(HistoryResolution::Raw).unwrap(), 2);
|
||||
assert_eq!(store.count(HistoryResolution::Minute).unwrap(), 1);
|
||||
drop(store);
|
||||
|
||||
let history = DiagnosticStore::history(&path, HistoryResolution::Raw, 0, 4_000).unwrap();
|
||||
assert_eq!(history.samples, vec![sample(3_600), sample(3_610)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_deletes_only_expired_resolution_rows() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let path = directory.path().join("diagnostics.sqlite3");
|
||||
let config = DiagnosticsConfig {
|
||||
raw_retention_hours: 1,
|
||||
minute_retention_days: 1,
|
||||
..DiagnosticsConfig::default()
|
||||
};
|
||||
let mut store = DiagnosticStore::open(&path).unwrap();
|
||||
store.record(&sample(1), &config).unwrap();
|
||||
store.record(&sample(90_000), &config).unwrap();
|
||||
assert_eq!(store.count(HistoryResolution::Raw).unwrap(), 1);
|
||||
assert_eq!(store.count(HistoryResolution::Minute).unwrap(), 1);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
// 负责把相同内容的多个 infohash 聚合为稳定且可排序的搜索内容组
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use super::{Availability, AvailabilityStatus, Heat, InfoHash, TorrentRecord};
|
||||
use super::{Availability, Heat, TorrentRecord};
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
use super::{AvailabilityStatus, InfoHash};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ContentGroup {
|
||||
@@ -17,6 +20,7 @@ pub struct ContentGroup {
|
||||
pub availability: Availability,
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
pub(crate) struct ContentGroupBuilder {
|
||||
content_key: [u8; 32],
|
||||
now: u64,
|
||||
@@ -31,6 +35,7 @@ pub(crate) struct ContentGroupBuilder {
|
||||
availability_initialized: bool,
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
impl ContentGroupBuilder {
|
||||
pub(crate) fn new(content_key: [u8; 32], now: u64) -> Self {
|
||||
Self {
|
||||
@@ -116,8 +121,10 @@ impl ContentGroupBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
type RepresentativeRank = (u8, u8, u32, u64, u64, std::cmp::Reverse<InfoHash>);
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
fn representative_rank(record: &TorrentRecord, heat: Heat) -> RepresentativeRank {
|
||||
let availability = match record.availability.status {
|
||||
AvailabilityStatus::Active => 2,
|
||||
@@ -134,6 +141,7 @@ fn representative_rank(record: &TorrentRecord, heat: Heat) -> RepresentativeRank
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
fn merge_availability(target: &mut Availability, candidate: &Availability) {
|
||||
if availability_rank(candidate.status) > availability_rank(target.status) {
|
||||
target.status = candidate.status;
|
||||
@@ -148,6 +156,7 @@ fn merge_availability(target: &mut Availability, candidate: &Availability) {
|
||||
target.next_check_at = target.next_check_at.max(candidate.next_check_at);
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
fn availability_rank(status: AvailabilityStatus) -> u8 {
|
||||
match status {
|
||||
AvailabilityStatus::Active => 2,
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use dht_crawler::TorrentInfo;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
@@ -13,6 +12,17 @@ use super::{
|
||||
const MAX_STORED_PEERS: usize = 32;
|
||||
const METADATA_VALIDATION_VERSION: u64 = 1;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MetadataCandidate {
|
||||
pub info_hash: String,
|
||||
pub name: String,
|
||||
pub total_size: u64,
|
||||
pub files: Vec<TorrentFile>,
|
||||
pub piece_length: u64,
|
||||
pub source_peers: Vec<String>,
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct MetadataLimits {
|
||||
pub max_files: usize,
|
||||
@@ -164,7 +174,7 @@ impl TorrentRecordError {
|
||||
}
|
||||
|
||||
pub(crate) fn into_record(
|
||||
info: TorrentInfo,
|
||||
info: MetadataCandidate,
|
||||
limits: MetadataLimits,
|
||||
) -> Result<TorrentRecord, TorrentRecordError> {
|
||||
let info_hash = InfoHash::from_str(&info.info_hash)?;
|
||||
@@ -181,14 +191,7 @@ pub(crate) fn into_record(
|
||||
for file in &info.files {
|
||||
validate_path(&file.path, limits)?;
|
||||
}
|
||||
let files: Vec<_> = info
|
||||
.files
|
||||
.into_iter()
|
||||
.map(|file| TorrentFile {
|
||||
path: file.path,
|
||||
size: file.size,
|
||||
})
|
||||
.collect();
|
||||
let files = info.files;
|
||||
let calculated_size = files.iter().try_fold(0_u64, |total, file| {
|
||||
total
|
||||
.checked_add(file.size)
|
||||
@@ -201,7 +204,7 @@ pub(crate) fn into_record(
|
||||
});
|
||||
}
|
||||
let content_key = content_key(&files)?;
|
||||
let mut source_peers = info.peers;
|
||||
let mut source_peers = info.source_peers;
|
||||
source_peers.sort_unstable();
|
||||
source_peers.dedup();
|
||||
source_peers.truncate(MAX_STORED_PEERS);
|
||||
@@ -278,31 +281,28 @@ fn validate_path(path: &str, limits: MetadataLimits) -> Result<(), TorrentRecord
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl TryFrom<TorrentInfo> for TorrentRecord {
|
||||
impl TryFrom<MetadataCandidate> for TorrentRecord {
|
||||
type Error = TorrentRecordError;
|
||||
|
||||
fn try_from(info: TorrentInfo) -> Result<Self, Self::Error> {
|
||||
fn try_from(info: MetadataCandidate) -> Result<Self, Self::Error> {
|
||||
into_record(info, MetadataLimits::default())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use dht_crawler::FileInfo;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn torrent_info(files: Vec<FileInfo>) -> TorrentInfo {
|
||||
TorrentInfo {
|
||||
fn metadata_candidate(files: Vec<TorrentFile>) -> MetadataCandidate {
|
||||
MetadataCandidate {
|
||||
info_hash: "0101010101010101010101010101010101010101".into(),
|
||||
magnet_link: String::new(),
|
||||
name: "Example".into(),
|
||||
total_size: files
|
||||
.iter()
|
||||
.fold(0_u64, |total, file| total.saturating_add(file.size)),
|
||||
files,
|
||||
piece_length: 16_384,
|
||||
peers: Vec::new(),
|
||||
source_peers: Vec::new(),
|
||||
timestamp: 1,
|
||||
}
|
||||
}
|
||||
@@ -315,17 +315,17 @@ mod tests {
|
||||
max_path_bytes: 8,
|
||||
max_path_depth: 2,
|
||||
};
|
||||
let accepted = torrent_info(vec![FileInfo {
|
||||
let accepted = metadata_candidate(vec![TorrentFile {
|
||||
path: "dir/a.rs".into(),
|
||||
size: 1,
|
||||
}]);
|
||||
assert!(into_record(accepted, limits).is_ok());
|
||||
let too_many = torrent_info(vec![
|
||||
FileInfo {
|
||||
let too_many = metadata_candidate(vec![
|
||||
TorrentFile {
|
||||
path: "a".into(),
|
||||
size: 1,
|
||||
},
|
||||
FileInfo {
|
||||
TorrentFile {
|
||||
path: "b".into(),
|
||||
size: 1,
|
||||
},
|
||||
@@ -350,7 +350,7 @@ mod tests {
|
||||
("a/b/c", MetadataRejectionReason::PathTooDeep),
|
||||
("a\0b", MetadataRejectionReason::InvalidPath),
|
||||
] {
|
||||
let info = torrent_info(vec![FileInfo {
|
||||
let info = metadata_candidate(vec![TorrentFile {
|
||||
path: path.into(),
|
||||
size: 1,
|
||||
}]);
|
||||
@@ -363,12 +363,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn size_overflow_is_rejected_without_panicking() {
|
||||
let mut info = torrent_info(vec![
|
||||
FileInfo {
|
||||
let mut info = metadata_candidate(vec![
|
||||
TorrentFile {
|
||||
path: "a".into(),
|
||||
size: u64::MAX,
|
||||
},
|
||||
FileInfo {
|
||||
TorrentFile {
|
||||
path: "b".into(),
|
||||
size: 1,
|
||||
},
|
||||
@@ -12,10 +12,14 @@ pub use content_filter::{
|
||||
FileMatchKind, FileRuleAction, FilterOutcome,
|
||||
};
|
||||
pub use content_group::ContentGroup;
|
||||
#[cfg(any(feature = "rocksdb-storage", test))]
|
||||
pub(crate) use content_group::ContentGroupBuilder;
|
||||
pub use fingerprint::content_key;
|
||||
pub use info_hash::InfoHash;
|
||||
pub use metadata::{MetadataLimits, MetadataRejectionReason, RejectedMetadata, TorrentRecordError};
|
||||
pub use metadata::{
|
||||
MetadataCandidate, MetadataLimits, MetadataRejectionReason, RejectedMetadata,
|
||||
TorrentRecordError,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use torrent::test_record;
|
||||
pub use torrent::{
|
||||
@@ -1,11 +1,10 @@
|
||||
// 负责定义种子记录活跃度可用性和重复发现时的状态演进
|
||||
|
||||
use dht_crawler::TorrentInfo;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[cfg(test)]
|
||||
use super::content_key;
|
||||
use super::{InfoHash, MetadataLimits, TorrentRecordError};
|
||||
use super::{InfoHash, MetadataCandidate, MetadataLimits, TorrentRecordError};
|
||||
|
||||
const MAX_STORED_PEERS: usize = 32;
|
||||
const ACTIVITY_SCALE: u64 = 1_000;
|
||||
@@ -126,7 +125,7 @@ impl TorrentRecord {
|
||||
}
|
||||
|
||||
pub fn try_from_with_limits(
|
||||
info: TorrentInfo,
|
||||
info: MetadataCandidate,
|
||||
limits: MetadataLimits,
|
||||
) -> Result<Self, TorrentRecordError> {
|
||||
super::metadata::into_record(info, limits)
|
||||
@@ -249,8 +248,6 @@ pub(crate) fn test_record(hash_byte: u8, timestamp: u64) -> TorrentRecord {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use dht_crawler::FileInfo;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
@@ -296,17 +293,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn freshly_downloaded_metadata_is_immediately_active() {
|
||||
let record = TorrentRecord::try_from(TorrentInfo {
|
||||
let record = TorrentRecord::try_from(MetadataCandidate {
|
||||
info_hash: "0101010101010101010101010101010101010101".into(),
|
||||
magnet_link: String::new(),
|
||||
name: "Example".into(),
|
||||
total_size: 42,
|
||||
files: vec![FileInfo {
|
||||
files: vec![TorrentFile {
|
||||
path: "example.bin".into(),
|
||||
size: 42,
|
||||
}],
|
||||
piece_length: 16_384,
|
||||
peers: vec!["127.0.0.1:6881".into()],
|
||||
source_peers: vec!["127.0.0.1:6881".into()],
|
||||
timestamp: 100,
|
||||
})
|
||||
.unwrap();
|
||||
@@ -1,27 +1,15 @@
|
||||
// 负责组装应用依赖启动运行时并协调服务生命周期
|
||||
// 负责加载启动配置初始化日志并选择正常运行或离线恢复模式
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
mod api;
|
||||
mod app;
|
||||
mod backup;
|
||||
mod config;
|
||||
mod crawler;
|
||||
mod disk_guard;
|
||||
mod error;
|
||||
mod index_worker;
|
||||
mod monitor;
|
||||
mod shutdown;
|
||||
mod telemetry;
|
||||
mod verification;
|
||||
use crate::{backup, config, error, telemetry};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
pub async fn run_cli() -> std::process::ExitCode {
|
||||
let startup = match config::Cli::parse().load() {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
eprintln!("无法加载配置: {error}");
|
||||
std::process::exit(1);
|
||||
return std::process::ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let restore_requested = startup.restore_checkpoint.is_some();
|
||||
@@ -34,7 +22,7 @@ async fn main() {
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
eprintln!("无法初始化日志: {error}");
|
||||
std::process::exit(1);
|
||||
return std::process::ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let result = if let Some(checkpoint) = startup.restore_checkpoint {
|
||||
@@ -52,10 +40,13 @@ async fn main() {
|
||||
})
|
||||
.map_err(error::AppError::from)
|
||||
} else {
|
||||
app::run(startup.app).await
|
||||
crate::app::run(startup.app, startup.config_service).await
|
||||
};
|
||||
if let Err(error) = result {
|
||||
tracing::error!(%error, "dht-search 退出");
|
||||
std::process::exit(1);
|
||||
match result {
|
||||
Ok(()) => std::process::ExitCode::SUCCESS,
|
||||
Err(error) => {
|
||||
tracing::error!(%error, "dht-search 退出");
|
||||
std::process::ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,10 @@ pub(crate) enum AppError {
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("配置解析失败: {0}")]
|
||||
Toml(#[from] toml::de::Error),
|
||||
#[error("配置序列化失败: {0}")]
|
||||
TomlSerialize(#[from] toml::ser::Error),
|
||||
#[error("内容过滤配置无效: {0}")]
|
||||
ContentFilter(#[from] dht_search::domain::ContentFilterError),
|
||||
ContentFilter(#[from] crate::domain::ContentFilterError),
|
||||
#[error("备份或恢复失败: {0}")]
|
||||
Backup(#[from] crate::backup::BackupError),
|
||||
#[error("配置无效: {0}")]
|
||||
@@ -15,9 +17,9 @@ pub(crate) enum AppError {
|
||||
#[error("DHT 服务失败: {0}")]
|
||||
Dht(#[from] dht_crawler::DHTError),
|
||||
#[error("存储失败: {0}")]
|
||||
Storage(#[from] dht_search::storage::StorageError),
|
||||
Storage(#[from] crate::storage::StorageError),
|
||||
#[error("搜索失败: {0}")]
|
||||
Search(#[from] dht_search::search::SearchError),
|
||||
Search(#[from] crate::search::SearchError),
|
||||
#[error("持久化 worker 异常退出")]
|
||||
PersistenceWorkerPanicked,
|
||||
#[error("持久化 worker 失败: {0}")]
|
||||
@@ -26,4 +28,6 @@ pub(crate) enum AppError {
|
||||
IndexWorker(String),
|
||||
#[error("可用性验证 worker 失败: {0}")]
|
||||
VerificationWorker(String),
|
||||
#[error("运行诊断失败: {0}")]
|
||||
Diagnostics(String),
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
// 负责导出可测试核心能力并组合应用运行入口
|
||||
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod api;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod app;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod backup;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod config;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod crawler;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod diagnostics;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod disk_guard;
|
||||
pub mod domain;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod entry;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod error;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod index_worker;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod monitor;
|
||||
pub mod search;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod shutdown;
|
||||
pub mod storage;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod telemetry;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod verification;
|
||||
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
pub use entry::run_cli;
|
||||
@@ -0,0 +1,6 @@
|
||||
// 负责进入异步运行时并把退出状态返回给操作系统
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> std::process::ExitCode {
|
||||
dht_search::run_cli().await
|
||||
}
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::domain::MetadataRejectionReason;
|
||||
use dht_crawler::DHTServer;
|
||||
use dht_search::domain::MetadataRejectionReason;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::{crawler::pipeline::PersistenceIngress, disk_guard::DiskGuard};
|
||||
@@ -2,7 +2,11 @@
|
||||
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering as AtomicOrdering},
|
||||
},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use crate::domain::ContentGroup;
|
||||
@@ -31,10 +35,26 @@ pub struct SearchEngine {
|
||||
inner: Arc<SearchInner>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct SearchDiagnostics {
|
||||
pub documents: u64,
|
||||
pub writer_memory_budget_bytes: u64,
|
||||
pub commits: u64,
|
||||
pub commit_failures: u64,
|
||||
pub last_commit_at: Option<u64>,
|
||||
pub last_commit_duration_millis: u64,
|
||||
pub last_commit_documents: u64,
|
||||
}
|
||||
|
||||
struct SearchInner {
|
||||
reader: IndexReader,
|
||||
writer: Mutex<IndexWriter>,
|
||||
fields: SearchFields,
|
||||
commits: AtomicU64,
|
||||
commit_failures: AtomicU64,
|
||||
last_commit_at: AtomicU64,
|
||||
last_commit_duration_millis: AtomicU64,
|
||||
last_commit_documents: AtomicU64,
|
||||
}
|
||||
|
||||
impl SearchEngine {
|
||||
@@ -94,6 +114,11 @@ impl SearchEngine {
|
||||
reader,
|
||||
writer: Mutex::new(writer),
|
||||
fields,
|
||||
commits: AtomicU64::new(0),
|
||||
commit_failures: AtomicU64::new(0),
|
||||
last_commit_at: AtomicU64::new(0),
|
||||
last_commit_duration_millis: AtomicU64::new(0),
|
||||
last_commit_documents: AtomicU64::new(0),
|
||||
}),
|
||||
},
|
||||
created,
|
||||
@@ -104,28 +129,69 @@ impl SearchEngine {
|
||||
if groups.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let started = Instant::now();
|
||||
let fields = self.inner.fields;
|
||||
let mut writer = self
|
||||
.inner
|
||||
.writer
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for group in groups {
|
||||
writer.delete_term(Term::from_field_text(
|
||||
fields.content_key,
|
||||
&hex::encode(group.content_key),
|
||||
));
|
||||
writer.add_document(super::document::from_group(group, fields))?;
|
||||
let result = (|| {
|
||||
let mut writer = self
|
||||
.inner
|
||||
.writer
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for group in groups {
|
||||
writer.delete_term(Term::from_field_text(
|
||||
fields.content_key,
|
||||
&hex::encode(group.content_key),
|
||||
));
|
||||
writer.add_document(super::document::from_group(group, fields))?;
|
||||
}
|
||||
writer.commit()?;
|
||||
self.inner.reader.reload()?;
|
||||
Ok(())
|
||||
})();
|
||||
self.inner.last_commit_duration_millis.store(
|
||||
started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64,
|
||||
AtomicOrdering::Relaxed,
|
||||
);
|
||||
if result.is_ok() {
|
||||
self.inner.commits.fetch_add(1, AtomicOrdering::Relaxed);
|
||||
self.inner
|
||||
.last_commit_at
|
||||
.store(unix_timestamp(), AtomicOrdering::Relaxed);
|
||||
self.inner.last_commit_documents.store(
|
||||
groups.len().min(u64::MAX as usize) as u64,
|
||||
AtomicOrdering::Relaxed,
|
||||
);
|
||||
} else {
|
||||
self.inner
|
||||
.commit_failures
|
||||
.fetch_add(1, AtomicOrdering::Relaxed);
|
||||
}
|
||||
writer.commit()?;
|
||||
self.inner.reader.reload()?;
|
||||
Ok(())
|
||||
result
|
||||
}
|
||||
|
||||
pub fn num_docs(&self) -> u64 {
|
||||
self.inner.reader.searcher().num_docs()
|
||||
}
|
||||
|
||||
pub fn diagnostics(&self) -> SearchDiagnostics {
|
||||
let last_commit_at = self.inner.last_commit_at.load(AtomicOrdering::Relaxed);
|
||||
SearchDiagnostics {
|
||||
documents: self.num_docs(),
|
||||
writer_memory_budget_bytes: INDEX_WRITER_MEMORY_BYTES as u64,
|
||||
commits: self.inner.commits.load(AtomicOrdering::Relaxed),
|
||||
commit_failures: self.inner.commit_failures.load(AtomicOrdering::Relaxed),
|
||||
last_commit_at: (last_commit_at != 0).then_some(last_commit_at),
|
||||
last_commit_duration_millis: self
|
||||
.inner
|
||||
.last_commit_duration_millis
|
||||
.load(AtomicOrdering::Relaxed),
|
||||
last_commit_documents: self
|
||||
.inner
|
||||
.last_commit_documents
|
||||
.load(AtomicOrdering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index_pending(
|
||||
&self,
|
||||
repository: &dyn TorrentRepository,
|
||||
@@ -263,7 +329,6 @@ fn sorted_documents(
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn unix_timestamp() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -6,7 +6,7 @@ mod indexer;
|
||||
mod query;
|
||||
mod schema;
|
||||
|
||||
pub use indexer::SearchEngine;
|
||||
pub use indexer::{SearchDiagnostics, SearchEngine};
|
||||
pub use query::{
|
||||
AvailabilitySummary, SearchHit, SearchMode, SearchOptions, SearchPage, SearchSort,
|
||||
};
|
||||
@@ -1,13 +1,15 @@
|
||||
// 负责暴露持久化抽象并隐藏 RocksDB 的具体实现细节
|
||||
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod keys;
|
||||
mod repository;
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
mod rocks;
|
||||
|
||||
pub use repository::{
|
||||
CheckpointSummary, ContentGroupTask, ContentVariants, StorageError, TorrentRepository,
|
||||
UpsertOutcome, VerificationEnqueueOutcome, VerificationPriority, VerificationRequest,
|
||||
CheckpointSummary, ContentGroupTask, ContentVariants, StorageDiagnostics, StorageError,
|
||||
TorrentRepository, UpsertOutcome, VerificationEnqueueOutcome, VerificationPriority,
|
||||
VerificationRequest,
|
||||
};
|
||||
#[cfg(feature = "rocksdb-storage")]
|
||||
pub use rocks::RocksTorrentRepository;
|
||||
@@ -97,6 +97,16 @@ pub struct CheckpointSummary {
|
||||
pub records: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct StorageDiagnostics {
|
||||
pub block_cache_bytes: Option<u64>,
|
||||
pub memtable_bytes: Option<u64>,
|
||||
pub pending_compaction_bytes: Option<u64>,
|
||||
pub live_sst_bytes: Option<u64>,
|
||||
pub running_compactions: Option<u64>,
|
||||
pub estimated_keys: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum VerificationPriority {
|
||||
Normal,
|
||||
@@ -27,8 +27,9 @@ use super::{
|
||||
verification_locator_key, verification_task_key, verification_task_prefix,
|
||||
},
|
||||
repository::{
|
||||
CheckpointSummary, ContentGroupTask, ContentVariants, StorageError, TorrentRepository,
|
||||
UpsertOutcome, VerificationEnqueueOutcome, VerificationPriority, VerificationRequest,
|
||||
CheckpointSummary, ContentGroupTask, ContentVariants, StorageDiagnostics, StorageError,
|
||||
TorrentRepository, UpsertOutcome, VerificationEnqueueOutcome, VerificationPriority,
|
||||
VerificationRequest,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -98,6 +99,23 @@ impl RocksTorrentRepository {
|
||||
self.content_filter_changed
|
||||
}
|
||||
|
||||
pub fn diagnostics(&self) -> Result<StorageDiagnostics, StorageError> {
|
||||
Ok(StorageDiagnostics {
|
||||
block_cache_bytes: self.db.property_int_value("rocksdb.block-cache-usage")?,
|
||||
memtable_bytes: self
|
||||
.db
|
||||
.property_int_value("rocksdb.cur-size-all-mem-tables")?,
|
||||
pending_compaction_bytes: self
|
||||
.db
|
||||
.property_int_value("rocksdb.estimate-pending-compaction-bytes")?,
|
||||
live_sst_bytes: self.db.property_int_value("rocksdb.live-sst-files-size")?,
|
||||
running_compactions: self
|
||||
.db
|
||||
.property_int_value("rocksdb.num-running-compactions")?,
|
||||
estimated_keys: self.db.property_int_value("rocksdb.estimate-num-keys")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn create_checkpoint(&self, path: impl AsRef<Path>) -> Result<(), StorageError> {
|
||||
Checkpoint::new(&self.db)?.create_checkpoint(path)?;
|
||||
Ok(())
|
||||
@@ -7,11 +7,11 @@ use std::sync::{
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::{collections::HashSet, net::SocketAddr};
|
||||
|
||||
use dht_crawler::DHTServer;
|
||||
use dht_search::{
|
||||
use crate::{
|
||||
domain::{InfoHash, VerificationResult},
|
||||
storage::{RocksTorrentRepository, TorrentRepository, VerificationPriority},
|
||||
};
|
||||
use dht_crawler::DHTServer;
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -125,7 +125,7 @@ impl VerificationIngress {
|
||||
let outcome =
|
||||
repository.enqueue_verification(&hashes, priority, unix_timestamp(), capacity)?;
|
||||
let queue_len = repository.verification_queue_len()?;
|
||||
Ok::<_, dht_search::storage::StorageError>((outcome, queue_len))
|
||||
Ok::<_, crate::storage::StorageError>((outcome, queue_len))
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
@@ -2,11 +2,10 @@
|
||||
|
||||
#![cfg(feature = "rocksdb-storage")]
|
||||
|
||||
use dht_crawler::{FileInfo, TorrentInfo};
|
||||
use dht_search::{
|
||||
domain::{
|
||||
ContentFilter, ContentFilterConfig, FileFilterRule, FileMatchField, FileMatchKind,
|
||||
FileRuleAction, TorrentRecord,
|
||||
FileRuleAction, MetadataCandidate, TorrentFile, TorrentRecord,
|
||||
},
|
||||
search::SearchEngine,
|
||||
storage::{RocksTorrentRepository, TorrentRepository, UpsertOutcome},
|
||||
@@ -19,17 +18,16 @@ fn public_components_compose_into_a_restart_safe_search_flow() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let rocksdb = directory.path().join("rocksdb");
|
||||
let tantivy = directory.path().join("tantivy");
|
||||
let record = TorrentRecord::try_from(TorrentInfo {
|
||||
let record = TorrentRecord::try_from(MetadataCandidate {
|
||||
info_hash: "1212121212121212121212121212121212121212".into(),
|
||||
magnet_link: String::new(),
|
||||
name: "Public API 测试资源".into(),
|
||||
total_size: 42,
|
||||
files: vec![FileInfo {
|
||||
files: vec![TorrentFile {
|
||||
path: "docs/public-api.txt".into(),
|
||||
size: 42,
|
||||
}],
|
||||
piece_length: 16_384,
|
||||
peers: Vec::new(),
|
||||
source_peers: Vec::new(),
|
||||
timestamp: 100,
|
||||
})
|
||||
.unwrap();
|
||||
@@ -76,23 +74,22 @@ fn content_filter_excludes_padding_from_search_and_visible_details() {
|
||||
filter,
|
||||
)
|
||||
.unwrap();
|
||||
let record = TorrentRecord::try_from(TorrentInfo {
|
||||
let record = TorrentRecord::try_from(MetadataCandidate {
|
||||
info_hash: "3434343434343434343434343434343434343434".into(),
|
||||
magnet_link: String::new(),
|
||||
name: "Filtered Movie".into(),
|
||||
total_size: 142,
|
||||
files: vec![
|
||||
FileInfo {
|
||||
TorrentFile {
|
||||
path: "movie.mkv".into(),
|
||||
size: 42,
|
||||
},
|
||||
FileInfo {
|
||||
TorrentFile {
|
||||
path: "_____padding_file_1_请升级____".into(),
|
||||
size: 100,
|
||||
},
|
||||
],
|
||||
piece_length: 16_384,
|
||||
peers: Vec::new(),
|
||||
source_peers: Vec::new(),
|
||||
timestamp: 100,
|
||||
})
|
||||
.unwrap();
|
||||
@@ -8,20 +8,20 @@
|
||||
|
||||
```powershell
|
||||
$env:LIBCLANG_PATH = "D:\tools\dht\.tools\libclang\clang\native"
|
||||
cargo run -p dht-search -- --config dht-search.example.toml
|
||||
cargo run -p dht-search --bin dht-search -- --config dht-search.toml
|
||||
```
|
||||
|
||||
然后打开另一个终端启动前端开发服务
|
||||
|
||||
```powershell
|
||||
cd web
|
||||
cd src/web
|
||||
bun install
|
||||
bun run dev
|
||||
```
|
||||
|
||||
打开 `http://127.0.0.1:5173`
|
||||
|
||||
Vite 会将搜索详情状态等接口代理到 `http://127.0.0.1:8080`
|
||||
Vite 会将搜索详情状态诊断和配置接口代理到 `http://127.0.0.1:8080` 可以通过 `DHT_API_TARGET` 环境变量覆盖目标
|
||||
|
||||
## 检查和构建
|
||||
|
||||
@@ -29,18 +29,18 @@ Vite 会将搜索详情状态等接口代理到 `http://127.0.0.1:8080`
|
||||
bun run check
|
||||
```
|
||||
|
||||
生产静态资源输出到 `web/dist`
|
||||
生产静态资源输出到 `src/web/dist`
|
||||
|
||||
## 生产运行
|
||||
|
||||
在启动 Rust 服务前构建一次 Web 资源
|
||||
|
||||
```powershell
|
||||
cd web
|
||||
cd src/web
|
||||
bun install --frozen-lockfile
|
||||
bun run build
|
||||
cd ..
|
||||
cargo run --release -p dht-search -- --config dht-search.example.toml
|
||||
cd ../..
|
||||
cargo run --release -p dht-search --bin dht-search -- --config dht-search.toml
|
||||
```
|
||||
|
||||
打开 `http://127.0.0.1:8080`
|
||||
@@ -60,6 +60,10 @@ Axum 根据配置中的 `http.web_dir` 提供静态资源和单页回退 不需
|
||||
- 磁力链接打开和复制
|
||||
- DHT 采集 索引 持久化和验证运行状态
|
||||
- 每秒自动刷新的运行状态和点击外部关闭
|
||||
- 独立的进程 RocksDB Tantivy DHT 和队列诊断页面
|
||||
- 使用懒加载 ECharts 展示内存存储压力 Metadata 吞吐以及 HTTP 请求错误趋势
|
||||
- 按职责分组的完整配置查看编辑校验保存和重启提示
|
||||
- 搜索诊断配置使用互不冲突的单页路由
|
||||
- 加载 空结果 接口错误 重试和移动端适配
|
||||
|
||||
## 添加组件
|
||||
@@ -8,9 +8,12 @@
|
||||
"@lucide/vue": "^1.29.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"echarts": "^6.1.0",
|
||||
"reka-ui": "^2.10.1",
|
||||
"tailwind-merge": "^3.6.0",
|
||||
"vue": "^3.5.40",
|
||||
"vue-echarts": "^8.1.0",
|
||||
"vue-router": "^4.6.4",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.3.3",
|
||||
@@ -146,6 +149,8 @@
|
||||
|
||||
"@vue/compiler-ssr": ["@vue/compiler-ssr@3.5.41", "https://registry.npmmirror.com/@vue/compiler-ssr/-/compiler-ssr-3.5.41.tgz", { "dependencies": { "@vue/compiler-dom": "3.5.41", "@vue/shared": "3.5.41" } }, "sha512-U3v5OejKEGqOI0Wy0+Sz7hGuIFZHA4LSXzrNM3IMIeDyJEBBfTpX26n3SDgToRpP2bLc9FfI2j/kSgcJ8Emq5A=="],
|
||||
|
||||
"@vue/devtools-api": ["@vue/devtools-api@6.6.4", "https://registry.npmmirror.com/@vue/devtools-api/-/devtools-api-6.6.4.tgz", {}, "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g=="],
|
||||
|
||||
"@vue/language-core": ["@vue/language-core@3.3.9", "https://registry.npmmirror.com/@vue/language-core/-/language-core-3.3.9.tgz", { "dependencies": { "@volar/language-core": "2.4.28", "@vue/compiler-dom": "^3.5.0", "@vue/shared": "^3.5.0", "alien-signals": "^3.2.1", "muggle-string": "^0.4.1", "path-browserify": "^1.0.1", "picomatch": "^4.0.4" } }, "sha512-in/68oAa4BCtVY6n/nkuhLIkV8DHYd2UivedJ6cMZ6UYtlq9jaoaSNUBHYCVO44z3nKg7MdE5OBoHKt5SxeBKQ=="],
|
||||
|
||||
"@vue/reactivity": ["@vue/reactivity@3.5.41", "https://registry.npmmirror.com/@vue/reactivity/-/reactivity-3.5.41.tgz", { "dependencies": { "@vue/shared": "3.5.41" } }, "sha512-rznsqKM0np0x18EjzF8x88MpEhdNsffbvFbckLL5+oUKz1BxAImEmO7J1ArRYSyo6aQaVoBDp7jEkT91OOxydA=="],
|
||||
@@ -180,6 +185,8 @@
|
||||
|
||||
"detect-libc": ["detect-libc@2.1.2", "https://registry.npmmirror.com/detect-libc/-/detect-libc-2.1.2.tgz", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
|
||||
|
||||
"echarts": ["echarts@6.1.0", "https://registry.npmmirror.com/echarts/-/echarts-6.1.0.tgz", { "dependencies": { "tslib": "2.3.0", "zrender": "6.1.0" } }, "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA=="],
|
||||
|
||||
"enhanced-resolve": ["enhanced-resolve@5.24.5", "https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="],
|
||||
|
||||
"entities": ["entities@7.0.1", "https://registry.npmmirror.com/entities/-/entities-7.0.1.tgz", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
@@ -248,7 +255,7 @@
|
||||
|
||||
"tinyglobby": ["tinyglobby@0.2.17", "https://registry.npmmirror.com/tinyglobby/-/tinyglobby-0.2.17.tgz", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="],
|
||||
|
||||
"tslib": ["tslib@2.8.1", "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
"tslib": ["tslib@2.3.0", "https://registry.npmmirror.com/tslib/-/tslib-2.3.0.tgz", {}, "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg=="],
|
||||
|
||||
"tw-animate-css": ["tw-animate-css@1.4.0", "https://registry.npmmirror.com/tw-animate-css/-/tw-animate-css-1.4.0.tgz", {}, "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ=="],
|
||||
|
||||
@@ -264,8 +271,16 @@
|
||||
|
||||
"vue-demi": ["vue-demi@0.14.10", "https://registry.npmmirror.com/vue-demi/-/vue-demi-0.14.10.tgz", { "peerDependencies": { "@vue/composition-api": "^1.0.0-rc.1", "vue": "^3.0.0-0 || ^2.6.0" }, "optionalPeers": ["@vue/composition-api"], "bin": { "vue-demi-fix": "bin/vue-demi-fix.js", "vue-demi-switch": "bin/vue-demi-switch.js" } }, "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg=="],
|
||||
|
||||
"vue-echarts": ["vue-echarts@8.1.0", "https://registry.npmmirror.com/vue-echarts/-/vue-echarts-8.1.0.tgz", { "peerDependencies": { "echarts": "^6.0.0", "vue": "^3.3.0" } }, "sha512-/uJVwijy3M2vIZ0NcPDgdZVqgboc6zZtC/vERfESau0eFpkHOIujj0sIiMiLP4kztQrckMFnDYWrid+gLI+IOg=="],
|
||||
|
||||
"vue-router": ["vue-router@4.6.4", "https://registry.npmmirror.com/vue-router/-/vue-router-4.6.4.tgz", { "dependencies": { "@vue/devtools-api": "^6.6.4" }, "peerDependencies": { "vue": "^3.5.0" } }, "sha512-Hz9q5sa33Yhduglwz6g9skT8OBPii+4bFn88w6J+J4MfEo4KRRpmiNG/hHHkdbRFlLBOqxN8y8gf2Fb0MTUgVg=="],
|
||||
|
||||
"vue-tsc": ["vue-tsc@3.3.9", "https://registry.npmmirror.com/vue-tsc/-/vue-tsc-3.3.9.tgz", { "dependencies": { "@volar/typescript": "2.4.28", "@vue/language-core": "3.3.9" }, "peerDependencies": { "typescript": ">=5.0.0" }, "bin": { "vue-tsc": "bin/vue-tsc.js" } }, "sha512-TS3Y1ux/IRoE8OCP2PpACAeOseuIs0UvWrcr7u+w3PmfY+SlCfEf8zjrBgnQksHUgLpthi5vHlffcQTQTdPBZA=="],
|
||||
|
||||
"zrender": ["zrender@6.1.0", "https://registry.npmmirror.com/zrender/-/zrender-6.1.0.tgz", { "dependencies": { "tslib": "2.3.0" } }, "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ=="],
|
||||
|
||||
"@swc/helpers/tslib": ["tslib@2.8.1", "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"@tailwindcss/node/lightningcss": ["lightningcss@1.32.0", "https://registry.npmmirror.com/lightningcss/-/lightningcss-1.32.0.tgz", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="],
|
||||
|
||||
"@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.3", "https://registry.npmmirror.com/@emnapi/core/-/core-1.11.3.tgz", { "dependencies": { "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" }, "bundled": true }, "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg=="],
|
||||
@@ -280,6 +295,8 @@
|
||||
|
||||
"@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"aria-hidden/tslib": ["tslib@2.8.1", "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"@tailwindcss/node/lightningcss/lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "https://registry.npmmirror.com/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="],
|
||||
|
||||
"@tailwindcss/node/lightningcss/lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "https://registry.npmmirror.com/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="],
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user