diff --git a/.gitignore b/.gitignore index 12e6412..1479803 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ /data/ /data-filter-test/ /benchmark-data/ -/dht-search.toml **/*.rs.bk # IDE diff --git a/README.md b/README.md index eb3f8a9..8a1fd8d 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,7 @@ opencodes/ 不参与构建的参考项目 Rust 测试分层和默认验证命令见 [`TESTING.md`](TESTING.md) -应用配置模板见 [`dht-search.example.toml`](dht-search.example.toml) - -无效文件隐藏规则见 [`content-filters.toml`](content-filters.toml) +应用全部配置和无效文件隐藏规则统一位于 [`config.toml`](config.toml) 应用构建运行和 API 文档见 [`src/search/README.md`](src/search/README.md) @@ -31,6 +29,6 @@ Web 开发和构建方式见 [`src/web/README.md`](src/web/README.md) 开发环境可以直接运行 `scripts\run.bat` 在当前窗口同时启动 Rust 后端和 Web 前端 按一次 `Ctrl+C` 即可统一停止 -启动脚本使用本地 `dht-search.toml` 文件 文件不存在时会从 `dht-search.example.toml` 创建 因此 Web 配置页不会修改版本库中的模板 +启动脚本和 Web 配置页统一读取并保存根目录 `config.toml` 任一服务异常退出时启动脚本会清理 Cargo Bun 及其子进程树 避免遗留 Vite 或后端进程 diff --git a/TODOS.md b/TODOS.md index 10c2cf2..9a9c1e1 100644 --- a/TODOS.md +++ b/TODOS.md @@ -216,7 +216,7 @@ - [x] `/stats` 返回验证队列发现握手成功失败和拒绝指标 - [ ] 统计真实数据的 infohash 重复率和内容重复率 -- [x] 使用独立配置文件定义文件名和文件路径隐藏规则 +- [x] 在统一 `config.toml` 中定义文件名和文件路径隐藏规则 - [x] 支持精确前缀后缀包含通配符和正则匹配并限制规则复杂度 - [x] 保留 RocksDB 原始文件列表并为详情统计搜索和内容聚合生成有效内容视图 - [x] 使用规则指纹在配置变化时重算内容组并从 RocksDB 重建 Tantivy @@ -346,6 +346,7 @@ - [x] 增加配置查询完整校验原子保存并发修订和统一重启提示 - [x] 增加 Web 诊断页和配置管理页 - [x] 使用页签拆分配置分类并隐藏底层配置存储位置 +- [x] 将主配置和内容过滤规则合并为唯一 `config.toml` - [ ] 将用户可编辑配置的持久化适配器从 TOML 迁移到 SQLite 完成二十四小时持续运行并继续观察私有内存 Metadata 成功率候选队列深度和每条成功 Metadata 的网络成本 diff --git a/dht-search.example.toml b/config.toml similarity index 66% rename from dht-search.example.toml rename to config.toml index 51d27a6..aa0b4df 100644 --- a/dht-search.example.toml +++ b/config.toml @@ -1,9 +1,9 @@ -# 定义 dht-search 的推荐起始配置并作为用户配置模板 +# 定义 dht-search 的全部运行配置 data_dir = "data" -content_filter_file = "content-filters.toml" persistence_queue_capacity = 8192 stats_interval_secs = 10 +# run_duration_secs = 3600 index_batch_size = 1024 index_interval_millis = 5000 @@ -43,6 +43,39 @@ max_name_bytes = 1024 max_path_bytes = 4096 max_path_depth = 64 +[content_filter] +version = 1 + +[[content_filter.file_rules]] +id = "bitcomet-padding-file" +enabled = true +field = "file-name" +match = "prefix" +value = "_____padding_file_" +case_sensitive = false +action = "hide" +reason = "BitComet 分片边界填充文件" + +[[content_filter.file_rules]] +id = "generic-pad-directory" +enabled = true +field = "file-path" +match = "regex" +value = '(^|/)\.pad/' +case_sensitive = false +action = "hide" +reason = "客户端分片边界填充目录" + +[[content_filter.file_rules]] +id = "libtorrent-padding-directory" +enabled = true +field = "file-path" +match = "regex" +value = '(^|/)\.____padding_file/' +case_sensitive = false +action = "hide" +reason = "libtorrent 分片边界填充目录" + [dht] port = 12313 netmode = "ipv4-only" diff --git a/content-filters.toml b/content-filters.toml deleted file mode 100644 index 70d2e25..0000000 --- a/content-filters.toml +++ /dev/null @@ -1,33 +0,0 @@ -# 定义不参与搜索展示统计和内容聚合的无效文件规则 - -version = 1 - -[[file_rules]] -id = "bitcomet-padding-file" -enabled = true -field = "file-name" -match = "prefix" -value = "_____padding_file_" -case_sensitive = false -action = "hide" -reason = "BitComet 分片边界填充文件" - -[[file_rules]] -id = "generic-pad-directory" -enabled = true -field = "file-path" -match = "regex" -value = '(^|/)\.pad/' -case_sensitive = false -action = "hide" -reason = "客户端分片边界填充目录" - -[[file_rules]] -id = "libtorrent-padding-directory" -enabled = true -field = "file-path" -match = "regex" -value = '(^|/)\.____padding_file/' -case_sensitive = false -action = "hide" -reason = "libtorrent 分片边界填充目录" diff --git a/dht-search.filter-test.toml b/dht-search.filter-test.toml deleted file mode 100644 index 07f8de5..0000000 --- a/dht-search.filter-test.toml +++ /dev/null @@ -1,13 +0,0 @@ -data_dir = "data-filter-test" -run_duration_secs = 300 - -[metadata_limits] -max_metadata_bytes = 65536 -max_files = 10 -max_name_bytes = 64 -max_path_bytes = 128 -max_path_depth = 4 - -[http] -listen = "127.0.0.1:8080" -web_dir = "src/web/dist" diff --git a/scripts/run.bat b/scripts/run.bat index d88ca51..f1c73eb 100644 --- a/scripts/run.bat +++ b/scripts/run.bat @@ -29,14 +29,10 @@ if not exist "%LIBCLANG_PATH%\libclang.dll" ( exit /b 1 ) -if not exist "%PROJECT_ROOT%\dht-search.toml" ( - echo [INFO] Creating local configuration from template - copy /Y "%PROJECT_ROOT%\dht-search.example.toml" "%PROJECT_ROOT%\dht-search.toml" >nul - if errorlevel 1 ( - echo [ERROR] Failed to create dht-search.toml - pause - exit /b 1 - ) +if not exist "%PROJECT_ROOT%\config.toml" ( + echo [ERROR] config.toml was not found + pause + exit /b 1 ) if not exist "%WEB_DIR%\node_modules" ( @@ -70,7 +66,7 @@ powershell -NoProfile -ExecutionPolicy Bypass -Command ^ " }" ^ "}" ^ "try {" ^ - " $backend = Start-Process -FilePath 'cargo' -ArgumentList @('run','-p','dht-search','--bin','dht-search','--','--config','dht-search.toml') -WorkingDirectory $env:PROJECT_ROOT -NoNewWindow -PassThru;" ^ + " $backend = Start-Process -FilePath 'cargo' -ArgumentList @('run','-p','dht-search','--bin','dht-search','--','--config','config.toml') -WorkingDirectory $env:PROJECT_ROOT -NoNewWindow -PassThru;" ^ " $web = Start-Process -FilePath 'bun' -ArgumentList @('--bun','run','dev','--','--host','127.0.0.1') -WorkingDirectory $env:WEB_DIR -NoNewWindow -PassThru;" ^ " while (-not $backend.HasExited -and -not $web.HasExited) { Start-Sleep -Milliseconds 250 }" ^ "} finally {" ^ diff --git a/src/search/README.md b/src/search/README.md index d1e5b52..300c30d 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -18,15 +18,13 @@ cargo build -p dht-search --release ## 配置 -复制根目录的配置模板 +根目录 `config.toml` 包含全部运行配置和内容过滤规则 ```powershell -Copy-Item dht-search.example.toml dht-search.toml +cargo run -p dht-search -- --config config.toml ``` -相对 `data_dir` 以配置文件所在目录为基准解析 - -`content_filter_file` 指向独立的无效文件过滤配置 相对路径同样以主配置文件所在目录为基准解析 推荐直接使用根目录的 `content-filters.toml` +相对目录以 `config.toml` 所在目录为基准解析 也可以通过命令行覆盖数据目录和本次运行时长 @@ -40,7 +38,7 @@ cargo run -p dht-search -- --data-dir D:\data\dht-search --run-duration-secs 360 ### 当前运行模板网络配置 -`dht-search.example.toml` 默认使用经过本机一分钟资源测试的 Bitmagnet 等效激进配置 主动 DHT 查询仍共享 `max_outbound_queries_per_second` 总预算 所有队列保持有界 可以按设备和网络条件主动下调 +`config.toml` 默认使用经过本机一分钟资源测试的 Bitmagnet 等效激进配置 主动 DHT 查询仍共享 `max_outbound_queries_per_second` 总预算 所有队列保持有界 可以按设备和网络条件主动下调 | 配置项 | 运行模板值 | 作用 | |---|---:|---| @@ -138,7 +136,7 @@ RocksDB 是唯一权威数据源 应用使用 RocksDB 原生 Checkpoint API 在 ```powershell target\release\dht-search.exe ` - --config dht-search.example.toml ` + --config config.toml ` --restore-checkpoint data\backups\checkpoint-00000001775400000000 ``` @@ -191,18 +189,16 @@ SQLite 使用 WAL 和单独 writer 线程 原始采样超过 24 小时自动删 `/stats` 返回 `metadata_filtered` 总数以及 `metadata_filtered_*` 分类计数 Web 运行状态展示本次运行的过滤总数 -可以使用独立数据目录和严格限制运行五分钟测试 不会污染正式数据目录 +可以通过命令行覆盖独立数据目录并运行五分钟测试 不会污染正式数据目录 ```powershell -cargo run -p dht-search -- --config dht-search.filter-test.toml +cargo run -p dht-search -- --config config.toml --data-dir data-filter-test --run-duration-secs 300 Invoke-RestMethod http://127.0.0.1:8080/stats | ConvertTo-Json -Depth 5 ``` -严格测试配置仅用于观察过滤效果 不应作为正式采集配置 - ### 无效文件隐藏规则 -`content-filters.toml` 控制哪些文件不参与详情展示 搜索 文件数量 有效大小和内容聚合 默认规则会隐藏 BitComet `_____padding_file_` 文件以及 `.pad` 和 `.____padding_file` 填充目录 +`config.toml` 的 `content_filter` 区域控制哪些文件不参与详情展示 搜索 文件数量 有效大小和内容聚合 默认规则会隐藏 BitComet `_____padding_file_` 文件以及 `.pad` 和 `.____padding_file` 填充目录 RocksDB 始终保存完整原始 Metadata 隐藏规则不会删除文件或种子 修改或回滚规则后应用会根据规则指纹重新计算内容组并从 RocksDB 重建 Tantivy @@ -232,7 +228,7 @@ Linux 生产运行必须把文件描述符上限提高到至少 65536 ```bash prlimit --nofile=65536:65536 -- \ - /opt/dht-search/dht-search --config /opt/dht-search/dht-search.toml + /opt/dht-search/dht-search --config /opt/dht-search/config.toml ``` systemd 服务需要设置 diff --git a/src/search/src/api/mod.rs b/src/search/src/api/mod.rs index 4931671..d6d6706 100644 --- a/src/search/src/api/mod.rs +++ b/src/search/src/api/mod.rs @@ -259,6 +259,29 @@ mod tests { serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) .unwrap(); let original_revision = config_snapshot["revision"].as_str().unwrap().to_owned(); + let mut invalid_filter = config_snapshot["config"].clone(); + invalid_filter["content_filter"]["file_rules"][0]["match"] = serde_json::json!("regex"); + invalid_filter["content_filter"]["file_rules"][0]["value"] = serde_json::json!("("); + let response = app + .clone() + .oneshot( + Request::builder() + .method("PUT") + .uri("/config") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "revision": original_revision.clone(), + "config": invalid_filter, + })) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + config_snapshot["config"]["dht"]["port"] = serde_json::json!(22_313); let update = serde_json::json!({ "revision": original_revision, diff --git a/src/search/src/config.rs b/src/search/src/config.rs index fd94427..00fa615 100644 --- a/src/search/src/config.rs +++ b/src/search/src/config.rs @@ -25,7 +25,7 @@ pub(crate) use store::{ConfigStore, TomlConfigStore}; #[derive(Debug, Parser)] #[command(name = "dht-search", version, about = "DHT 元数据采集和搜索服务")] pub(crate) struct Cli { - #[arg(long, default_value = "dht-search.toml")] + #[arg(long, default_value = "config.toml")] config: PathBuf, #[arg(long)] data_dir: Option, @@ -128,6 +128,7 @@ mod tests { assert_eq!(decoded.http.listen, dto.http.listen); assert_eq!(decoded.logging.file_prefix, dto.logging.file_prefix); assert_eq!(decoded.diagnostics.database, dto.diagnostics.database); + assert_eq!(decoded.content_filter, dto.content_filter); } #[test] @@ -145,10 +146,6 @@ mod tests { .unwrap(); let config = startup.app; assert_eq!(config.data_dir, directory.path().join("state")); - assert_eq!( - config.content_filter_file, - directory.path().join("content-filters.toml") - ); assert_eq!(config.logging.directory, directory.path().join("data/logs")); assert_eq!( config.backup.directory, @@ -191,6 +188,14 @@ mod tests { assert!(matches!(resolve(dto), Err(AppError::Config(_)))); } + #[test] + fn invalid_embedded_content_filter_is_rejected() { + let mut dto = AppConfigDto::default(); + dto.content_filter.file_rules[0].match_kind = crate::domain::FileMatchKind::Regex; + dto.content_filter.file_rules[0].value = "(".to_owned(); + assert!(matches!(resolve(dto), Err(AppError::ContentFilter(_)))); + } + #[test] fn disk_resume_threshold_must_exceed_minimum() { let mut dto = AppConfigDto::default(); diff --git a/src/search/src/config/model.rs b/src/search/src/config/model.rs index 8a946bd..173e065 100644 --- a/src/search/src/config/model.rs +++ b/src/search/src/config/model.rs @@ -4,11 +4,15 @@ use std::{net::SocketAddr, path::PathBuf}; use serde::{Deserialize, Serialize}; +use crate::domain::{ + ContentFilterConfig, FileFilterRule, FileMatchField, FileMatchKind, FileRuleAction, +}; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct AppConfigDto { pub(crate) data_dir: PathBuf, - pub(crate) content_filter_file: PathBuf, + pub(crate) content_filter: ContentFilterConfig, pub(crate) persistence_queue_capacity: usize, pub(crate) stats_interval_secs: u64, pub(crate) run_duration_secs: Option, @@ -140,7 +144,7 @@ impl Default for AppConfigDto { fn default() -> Self { Self { data_dir: PathBuf::from("data"), - content_filter_file: PathBuf::from("content-filters.toml"), + content_filter: default_content_filter(), persistence_queue_capacity: 8_192, stats_interval_secs: 10, run_duration_secs: None, @@ -158,6 +162,44 @@ impl Default for AppConfigDto { } } +fn default_content_filter() -> ContentFilterConfig { + ContentFilterConfig { + version: 1, + file_rules: vec![ + FileFilterRule { + id: "bitcomet-padding-file".to_owned(), + enabled: true, + field: FileMatchField::FileName, + match_kind: FileMatchKind::Prefix, + value: "_____padding_file_".to_owned(), + case_sensitive: false, + action: FileRuleAction::Hide, + reason: "BitComet 分片边界填充文件".to_owned(), + }, + FileFilterRule { + id: "generic-pad-directory".to_owned(), + enabled: true, + field: FileMatchField::FilePath, + match_kind: FileMatchKind::Regex, + value: r"(^|/)\.pad/".to_owned(), + case_sensitive: false, + action: FileRuleAction::Hide, + reason: "客户端分片边界填充目录".to_owned(), + }, + FileFilterRule { + id: "libtorrent-padding-directory".to_owned(), + enabled: true, + field: FileMatchField::FilePath, + match_kind: FileMatchKind::Regex, + value: r"(^|/)\.____padding_file/".to_owned(), + case_sensitive: false, + action: FileRuleAction::Hide, + reason: "libtorrent 分片边界填充目录".to_owned(), + }, + ], + } +} + impl Default for MetadataLimitsConfig { fn default() -> Self { Self { diff --git a/src/search/src/config/runtime.rs b/src/search/src/config/runtime.rs index 390ce82..c5cbc8b 100644 --- a/src/search/src/config/runtime.rs +++ b/src/search/src/config/runtime.rs @@ -1,8 +1,8 @@ // 负责解析校验用户配置并生成应用可直接使用的运行时配置 -use std::{fs, ops::Deref, path::Path}; +use std::{ops::Deref, path::Path}; -use crate::domain::{ContentFilter, ContentFilterConfig, MetadataLimits}; +use crate::domain::{ContentFilter, MetadataLimits}; use dht_crawler::{ BootstrapOptions, CrawlOptions, DHTOptions, MetadataOptions, NetMode, PeerLookupOptions, PoolOptions, RateLimitOptions, SampleInfohashesOptions, SchedulerOptions, TargetOptions, @@ -18,7 +18,6 @@ pub(crate) struct AppConfig(AppConfigDto); impl AppConfig { pub(crate) fn resolve(mut dto: AppConfigDto, base: &Path) -> Result { dto.data_dir = resolve_path(base, dto.data_dir); - dto.content_filter_file = resolve_path(base, dto.content_filter_file); dto.logging.directory = resolve_path(base, dto.logging.directory); dto.backup.directory = resolve_path(base, dto.backup.directory); dto.diagnostics.database = resolve_path(base, dto.diagnostics.database); @@ -29,9 +28,7 @@ impl AppConfig { } pub(crate) fn content_filter(&self) -> Result { - let contents = fs::read_to_string(&self.content_filter_file)?; - let config = toml::from_str::(&contents)?; - ContentFilter::compile(config).map_err(AppError::from) + ContentFilter::compile(self.content_filter.clone()).map_err(AppError::from) } pub(crate) fn dht_options(&self) -> DHTOptions { @@ -98,6 +95,7 @@ impl AppConfig { } fn validate(&self) -> Result<(), AppError> { + ContentFilter::compile(self.content_filter.clone())?; if self.persistence_queue_capacity == 0 { return Err(AppError::Config( "persistence_queue_capacity 必须大于零".to_owned(), diff --git a/src/search/src/config/store.rs b/src/search/src/config/store.rs index 3493f74..31772e2 100644 --- a/src/search/src/config/store.rs +++ b/src/search/src/config/store.rs @@ -75,7 +75,7 @@ fn temporary_path(destination: &Path) -> PathBuf { let file_name = destination .file_name() .and_then(|name| name.to_str()) - .unwrap_or("dht-search.toml"); + .unwrap_or("config.toml"); destination.with_file_name(format!( ".{file_name}.tmp-{}-{sequence}", std::process::id() diff --git a/src/web/README.md b/src/web/README.md index 5511c05..0ae7535 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -8,7 +8,7 @@ ```powershell $env:LIBCLANG_PATH = "D:\tools\dht\.tools\libclang\clang\native" -cargo run -p dht-search --bin dht-search -- --config dht-search.toml +cargo run -p dht-search --bin dht-search -- --config config.toml ``` 然后打开另一个终端启动前端开发服务 @@ -40,7 +40,7 @@ cd src/web bun install --frozen-lockfile bun run build cd ../.. -cargo run --release -p dht-search --bin dht-search -- --config dht-search.toml +cargo run --release -p dht-search --bin dht-search -- --config config.toml ``` 打开 `http://127.0.0.1:8080` diff --git a/src/web/src/pages/ConfigPage.vue b/src/web/src/pages/ConfigPage.vue index 2496a67..bf91a4e 100644 --- a/src/web/src/pages/ConfigPage.vue +++ b/src/web/src/pages/ConfigPage.vue @@ -1,12 +1,12 @@