feat(search): add index space inspection tool

Dockerfile

@@ -34,6 +34,20 @@ RUN --mount=type=cache,id=dht-cargo-registry,target=/usr/local/cargo/registry,sh
     && install -Dm755 target/release/dht-search /out/dht-search
 
 
+FROM rust-builder AS index-tools-builder
+
+RUN --mount=type=cache,id=dht-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,id=dht-cargo-git,target=/usr/local/cargo/git,sharing=locked \
+    --mount=type=cache,id=dht-cargo-target,target=/build/target,sharing=locked \
+    cargo build --locked --release -p dht-search --bin dht-index-inspect \
+    && install -Dm755 target/release/dht-index-inspect /out/dht-index-inspect
+
+
+FROM scratch AS index-tools
+
+COPY --from=index-tools-builder /out/dht-index-inspect /dht-index-inspect
+
+
 FROM debian:trixie-slim AS runtime
 
 RUN apt-get update \

README.md

@@ -154,6 +154,29 @@ bun run build
 
 运行诊断中的 `index_refresh_scheduled` 和 `index_refresh_suppressed` 分别表示重复发现实际安排和被时间桶合并的索引刷新数 `index_documents_written` 和 `index_documents_skipped` 用于确认待索引任务最终是否产生 Tantivy 文档写入
 
+### 真实索引空间测量
+
+使用 `dht-index-inspect` 可以只读统计 `CURRENT` 指向的活动 Tantivy 索引 不会创建 writer 或修改索引
+
+```powershell
+cargo run -p dht-search --bin dht-index-inspect -- data/search-index
+```
+
+2026-08-11 在远端 624801 个活动文档上的测量覆盖 99.9998% 的索引文件字节 总索引为 14.71 GiB 其中物理文档 1036396 个 删除文档比例为 39.71% 按物理文档比例估算相同 Schema 全新重建约为 8.87 GiB
+
+| 字段或组件 | 占用 | 比例 |
+|---|---:|---:|
+| 文件名 N-Gram `file_names` | 9.02 GiB | 61.34% |
+| 别名 N-Gram `aliases` | 1.73 GiB | 11.78% |
+| 标题 N-Gram `name` | 1.73 GiB | 11.75% |
+| 已停用查询字段 `regex_text` | 0.83 GiB | 5.62% |
+| 精确文件名 `exact_file_names` | 0.64 GiB | 4.35% |
+| 路径分词 `files_text` | 0.30 GiB | 2.04% |
+| 全部 stored fields | 0.20 GiB | 1.39% |
+| 全部 fast fields | 0.01 GiB | 0.09% |
+
+空间瓶颈是文件名 N-Gram 不是 stored field fast field 热度或可用性字段 `regex_text` 已无搜索调用方可在下次 Schema 更新中删除 `aliases` 当前重复包含代表标题需要停止重复索引 精确文件名是否保留以及文件名 N-Gram 范围需要通过同一数据集对比搜索覆盖率后决定
+
 ## 相关文档
 
 - 当前实施状态和后续计划见 [`TODOS.md`](TODOS.md)

TODOS.md

@@ -22,8 +22,10 @@
 
 - [ ] 在远端真实数据上验证六小时动态状态分桶对 Tantivy 文档重写和删除比例的改善
 - [ ] 对比优化前后的每小时索引增长提交文档数和段合并压力
-- [ ] 测量标题别名文件名完整路径 N-Gram fast field 和 stored field 的实际空间占比
-- [ ] 根据测量结果设计尽量不降低常用搜索效果的索引精简方案
+- [ ] 在下次 Schema 更新中删除已无查询调用方且占真实索引 5.62% 的 `regex_text`
+- [ ] 停止在 `aliases` 中重复索引代表标题并验证不同 infohash 的真实别名仍可搜索
+- [ ] 使用同一数据集比较文件名 N-Gram `2..8` `2..4` `2..3` 和 Basic postings 的空间查询延迟与误匹配
+- [ ] 评估删除 `exact_file_names` 对精确文件名相关性排序的影响
 - [ ] 评估大型种子文件采样数量和路径文本预算对搜索覆盖率与索引体积的影响
 - [ ] 明确优化后的单文档平均占用重建峰值空间和预期最大可容纳种子数量
 
@@ -96,8 +98,8 @@
 
 ## 当前下一步
 
-先部署并观察六小时动态状态分桶 使用诊断中的安排刷新抑制刷新实际写入和跳过写入数量确认效果
+先根据真实字段占用使用同一数据集比较文件名 N-Gram 候选方案并确定精简 Schema
 
-完成真实运行对比后再测量 Tantivy 字段空间占用并确定索引精简方案
+同时继续观察六小时动态状态分桶 使用诊断中的安排刷新抑制刷新实际写入和跳过写入数量确认效果
 
 Schema 调整时同时加入动态热度所需字段和稳定排序键 最终只执行一次影子索引重建

src/search/Cargo.toml

@@ -57,3 +57,7 @@ required-features = ["rocksdb-storage"]
 name = "dht-benchmark"
 path = "src/bin/dht-benchmark.rs"
 required-features = ["rocksdb-storage"]
+
+[[bin]]
+name = "dht-index-inspect"
+path = "src/bin/dht-index-inspect.rs"

src/search/src/bin/dht-index-inspect.rs

@@ -0,0 +1,193 @@
+// 负责只读统计活动 Tantivy 索引的字段和组件空间占用
+
+use std::{
+    collections::BTreeMap,
+    error::Error,
+    fs, io,
+    path::{Component, Path, PathBuf},
+};
+
+use clap::Parser;
+use serde::Serialize;
+use tantivy::Index;
+
+#[derive(Debug, Parser)]
+#[command(about = "只读统计 Tantivy 索引空间占用")]
+struct Args {
+    #[arg(value_name = "INDEX_PATH")]
+    index_path: PathBuf,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct ComponentReport {
+    term_dictionary_bytes: u64,
+    postings_bytes: u64,
+    positions_bytes: u64,
+    fast_fields_bytes: u64,
+    fieldnorms_bytes: u64,
+    stored_fields_bytes: u64,
+    deletes_bytes: u64,
+    total_bytes: u64,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct FieldReport {
+    term_dictionary_bytes: u64,
+    postings_bytes: u64,
+    positions_bytes: u64,
+    fast_fields_bytes: u64,
+    fieldnorms_bytes: u64,
+    total_bytes: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct IndexReport {
+    index_path: PathBuf,
+    segments: usize,
+    live_documents: u64,
+    physical_documents: u64,
+    deleted_documents: u64,
+    filesystem_bytes: u64,
+    measured_bytes: u64,
+    unmeasured_bytes: u64,
+    components: ComponentReport,
+    fields: BTreeMap<String, FieldReport>,
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let args = Args::parse();
+    let index_path = resolve_index_path(&args.index_path)?;
+    let index = Index::open_in_dir(&index_path)?;
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
+    let usage = searcher.space_usage()?;
+    let mut components = ComponentReport::default();
+    let mut fields = BTreeMap::<String, FieldReport>::new();
+
+    for segment in usage.segments() {
+        components.term_dictionary_bytes += segment.termdict().total().get_bytes();
+        components.postings_bytes += segment.postings().total().get_bytes();
+        components.positions_bytes += segment.positions().total().get_bytes();
+        components.fast_fields_bytes += segment.fast_fields().total().get_bytes();
+        components.fieldnorms_bytes += segment.fieldnorms().total().get_bytes();
+        components.stored_fields_bytes += segment.store().total().get_bytes();
+        components.deletes_bytes += segment.deletes().get_bytes();
+        add_field_usage(&mut fields, segment.termdict(), |field, bytes| {
+            field.term_dictionary_bytes += bytes;
+        });
+        add_field_usage(&mut fields, segment.postings(), |field, bytes| {
+            field.postings_bytes += bytes;
+        });
+        add_field_usage(&mut fields, segment.positions(), |field, bytes| {
+            field.positions_bytes += bytes;
+        });
+        add_field_usage(&mut fields, segment.fast_fields(), |field, bytes| {
+            field.fast_fields_bytes += bytes;
+        });
+        add_field_usage(&mut fields, segment.fieldnorms(), |field, bytes| {
+            field.fieldnorms_bytes += bytes;
+        });
+    }
+    components.total_bytes = usage.total().get_bytes();
+    for field in fields.values_mut() {
+        field.total_bytes = field.term_dictionary_bytes
+            + field.postings_bytes
+            + field.positions_bytes
+            + field.fast_fields_bytes
+            + field.fieldnorms_bytes;
+    }
+    let live_documents = searcher.num_docs();
+    let physical_documents = searcher
+        .segment_readers()
+        .iter()
+        .map(|segment| u64::from(segment.max_doc()))
+        .sum();
+    let filesystem_bytes = directory_bytes(&index_path)?;
+    let report = IndexReport {
+        index_path,
+        segments: usage.segments().len(),
+        live_documents,
+        physical_documents,
+        deleted_documents: physical_documents.saturating_sub(live_documents),
+        filesystem_bytes,
+        measured_bytes: components.total_bytes,
+        unmeasured_bytes: filesystem_bytes.saturating_sub(components.total_bytes),
+        components,
+        fields,
+    };
+    serde_json::to_writer_pretty(io::stdout().lock(), &report)?;
+    println!();
+    Ok(())
+}
+
+fn add_field_usage(
+    reports: &mut BTreeMap<String, FieldReport>,
+    usage: &tantivy::space_usage::PerFieldSpaceUsage,
+    mut add: impl FnMut(&mut FieldReport, u64),
+) {
+    for field in usage.fields() {
+        add(
+            reports.entry(field.field_name().to_owned()).or_default(),
+            field.total().get_bytes(),
+        );
+    }
+}
+
+fn resolve_index_path(path: &Path) -> io::Result<PathBuf> {
+    let current_path = path.join("CURRENT");
+    if !current_path.is_file() {
+        return Ok(path.to_owned());
+    }
+    let generation = fs::read_to_string(current_path)?;
+    let generation = generation.trim();
+    let mut components = Path::new(generation).components();
+    if generation.is_empty()
+        || !matches!(components.next(), Some(Component::Normal(_)))
+        || components.next().is_some()
+    {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "CURRENT 包含无效的索引代名称",
+        ));
+    }
+    Ok(path.join("generations").join(generation))
+}
+
+fn directory_bytes(path: &Path) -> io::Result<u64> {
+    let mut total = 0_u64;
+    for entry in fs::read_dir(path)? {
+        let entry = entry?;
+        let file_type = entry.file_type()?;
+        if file_type.is_file() {
+            total = total.saturating_add(entry.metadata()?.len());
+        } else if file_type.is_dir() {
+            total = total.saturating_add(directory_bytes(&entry.path())?);
+        }
+    }
+    Ok(total)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::resolve_index_path;
+    use std::fs;
+    use tempfile::TempDir;
+
+    #[test]
+    fn root_path_resolves_current_generation() {
+        let directory = TempDir::new().unwrap();
+        fs::create_dir(directory.path().join("generations")).unwrap();
+        fs::write(directory.path().join("CURRENT"), "g-123\n").unwrap();
+        assert_eq!(
+            resolve_index_path(directory.path()).unwrap(),
+            directory.path().join("generations").join("g-123")
+        );
+    }
+
+    #[test]
+    fn current_generation_cannot_escape_index_root() {
+        let directory = TempDir::new().unwrap();
+        fs::write(directory.path().join("CURRENT"), "../outside").unwrap();
+        assert!(resolve_index_path(directory.path()).is_err());
+    }
+}