tools/dht
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(search): reduce index storage and memory

Browse Source
This commit is contained in:
chuan
2026-08-10 23:59:10 +08:00
parent 13d3a3b5d6
commit 418fee42c0
7 changed files with 185 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+14 -10
View File
@@ -127,26 +127,30 @@ bun run build
| 记录数 | 内容文档 | RocksDB 写入 | Tantivy 索引 | 总磁盘 | 峰值内存 |
|---:|---:|---:|---:|---:|---:|
| 10,000 | 9,000 | 149,584 条/秒 | 3,794 文档/秒 | 52.42 MiB | 86.21 MiB |
| 100,000 | 90,000 | 118,833 条/秒 | 3,261 文档/秒 | 452.65 MiB | 360.04 MiB |
| 1,000,000 | 900,000 | 102,115 条/秒 | 2,520 文档/秒 | 4.36 GiB | 1.70 GiB |
| 10,000 | 9,000 | 102,433 条/秒 | 9,050 文档/秒 | 21.88 MiB | 53.07 MiB |
| 100,000 | 90,000 | 86,452 条/秒 | 8,423 文档/秒 | 174.24 MiB | 182.23 MiB |
| 1,000,000 | 900,000 | 75,248 条/秒 | 5,435 文档/秒 | 1.49 GiB | 433.14 MiB |
百万级查询 P95
| 查询类型 | P95 |
|---|---:|
| 中文关键词 | 2.571 ms |
| 英文关键词 | 5.793 ms |
| 文件路径片段 | 0.110 ms |
| 精确 infohash | 0.008 ms |
| 有限状态正则 | 656.086 ms |
| 最近收录排序 | 3.346 ms |
| 大小扩展名过滤 | 6.451 ms |
| 中文关键词 | 4.774 ms |
| 英文关键词 | 7.746 ms |
| 文件路径片段 | 1.063 ms |
| 精确 infohash | 0.031 ms |
| 有限状态正则 | 675.803 ms |
| 最近收录排序 | 3.367 ms |
| 大小扩展名过滤 | 6.641 ms |
百万级基准中普通搜索过滤排序精确哈希索引吞吐磁盘和峰值内存均达到当前目标 大命中集合正则仍是继续扩大规模前最值得优化的查询路径
大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 正则和通配符继续使用完整规范化文本字段 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB
## 相关文档
- 当前实施状态和后续计划见 [`TODOS.md`](TODOS.md)
TODOS.md
+3
View File
@@ -153,6 +153,8 @@
- [x] 暴露种子内容组待索引数量和全量重建状态进度
- [x] 按文件大小为大型种子选择最多 2048 个文件并限制路径文本预算
- [x] 优化完整名称别名文件名路径的相关性权重并使用热度时间稳定同分结果
- [x] 将标题文件名和路径拆分为有限 N-Gram 与路径分词策略并移除无用位置索引
- [x] 搜索写入器按需创建使影子重建期间旧活动索引保持纯查询占用
### 验收标准
@@ -161,6 +163,7 @@
- [x] 索引过程中异常退出不会永久丢失文档
- [x] 全量重建期间旧索引继续提供完整旧结果且新数据在原子切换后可见
- [x] 百万级测试数据常用查询延迟达到 `README.md` 记录的目标
- [ ] 使用远端真实数据验证紧凑索引的最终体积重建峰值内存和稳态内存
## 阶段四 HTTP 搜索服务
src/search/src/search/document.rs
-3
View File
@@ -65,9 +65,6 @@ pub(crate) fn from_group(group: &ContentGroup, fields: SearchFields) -> TantivyD
}
document.add_text(fields.files_text, &path);
document.add_text(fields.regex_text, &path);
if file_name != path.as_str() {
document.add_text(fields.regex_text, file_name);
}
indexed_files.push(file);
}
for extension in extensions(&indexed_files) {
src/search/src/search/filter.rs
+79 -65
View File
@@ -34,7 +34,7 @@ pub(crate) fn prepare(
SearchMode::Text if has_wildcard_syntax(query_text) => {
wildcard_query(query_text, fields)?
}
SearchMode::Text => text_query(query_text, fields),
SearchMode::Text => text_query(query_text, fields)?,
SearchMode::Regex => regex_query(query_text, fields)?,
});
}
@@ -179,19 +179,19 @@ fn push_regex_literal(regex: &mut String, character: char) {
regex.push(character);
}
fn text_query(query: &str, fields: SearchFields) -> Box<dyn Query> {
fn text_query(query: &str, fields: SearchFields) -> Result<Box<dyn Query>, SearchError> {
let normalized = normalize_text(query.trim());
if normalized.len() == 40 && normalized.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Box::new(TermQuery::new(
return Ok(Box::new(TermQuery::new(
Term::from_field_text(fields.info_hash, &normalized),
IndexRecordOption::Basic,
));
)));
}
let terms = query_terms(query);
if terms.is_empty() {
return Box::new(AllQuery);
let parts: Vec<_> = normalized.split_whitespace().map(str::to_owned).collect();
if parts.is_empty() {
return Ok(Box::new(AllQuery));
}
let mut required = Vec::with_capacity(terms.len() + 1);
let mut required = Vec::with_capacity(parts.len() + 1);
let mut exact = Vec::<(Occur, Box<dyn Query>)>::new();
if let Some(field) = fields.exact_name {
exact.push((
@@ -229,49 +229,25 @@ fn text_query(query: &str, fields: SearchFields) -> Box<dyn Query> {
)),
));
}
for term in terms {
let alternatives: Vec<(Occur, Box<dyn Query>)> = vec![
(
for part in parts {
let mut alternatives: Vec<(Occur, Box<dyn Query>)> = Vec::with_capacity(4);
push_boosted_ngram(&mut alternatives, fields.name, &part, 1, 10, 4.0);
push_boosted_ngram(&mut alternatives, fields.aliases, &part, 1, 10, 2.0);
if let Some(field) = fields.file_names {
push_boosted_ngram(&mut alternatives, field, &part, 2, 8, 1.5);
}
if let Some(query) = path_query(fields.files_text, &part) {
alternatives.push((Occur::Should, query));
}
if alternatives.is_empty() {
alternatives.push((
Occur::Should,
Box::new(BoostQuery::new(
Box::new(TermQuery::new(
Term::from_field_text(fields.name, &term),
IndexRecordOption::WithFreqs,
)),
4.0,
)),
),
(
Occur::Should,
Box::new(BoostQuery::new(
Box::new(TermQuery::new(
Term::from_field_text(fields.aliases, &term),
IndexRecordOption::WithFreqs,
)),
2.0,
)),
),
(
Occur::Should,
Box::new(BoostQuery::new(
Box::new(TermQuery::new(
Term::from_field_text(
fields.file_names.unwrap_or(fields.files_text),
&term,
),
IndexRecordOption::WithFreqs,
)),
1.5,
)),
),
(
Occur::Should,
Box::new(TermQuery::new(
Term::from_field_text(fields.files_text, &term),
IndexRecordOption::WithFreqs,
)),
),
];
Box::new(RegexQuery::from_pattern(
&format!(".*{}.*", regex::escape(&part)),
fields.regex_text,
)?),
));
}
required.push((
Occur::Must,
Box::new(BooleanQuery::new(alternatives)) as Box<dyn Query>,
@@ -280,24 +256,62 @@ fn text_query(query: &str, fields: SearchFields) -> Box<dyn Query> {
if !exact.is_empty() {
required.push((Occur::Should, Box::new(BooleanQuery::new(exact))));
}
Box::new(BooleanQuery::new(required))
Ok(Box::new(BooleanQuery::new(required)))
}
fn query_terms(query: &str) -> Vec<String> {
normalize_text(query)
.split_whitespace()
.flat_map(|part| {
let chars: Vec<_> = part.chars().collect();
if chars.len() <= 20 {
vec![part.to_owned()]
} else {
chars
.windows(20)
.map(|window| window.iter().collect())
.collect()
}
fn push_boosted_ngram(
alternatives: &mut Vec<(Occur, Box<dyn Query>)>,
field: tantivy::schema::Field,
value: &str,
min_gram: usize,
max_gram: usize,
boost: f32,
) {
let characters: Vec<_> = value.chars().collect();
if characters.len() < min_gram {
return;
}
let chunks: Vec<String> = if characters.len() <= max_gram {
vec![value.to_owned()]
} else {
characters
.windows(max_gram)
.map(|window| window.iter().collect())
.collect()
};
let clauses = chunks
.into_iter()
.map(|chunk| {
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(field, &chunk),
IndexRecordOption::WithFreqs,
)) as Box<dyn Query>,
)
})
.collect()
.collect();
alternatives.push((
Occur::Should,
Box::new(BoostQuery::new(Box::new(BooleanQuery::new(clauses)), boost)),
));
}
fn path_query(field: tantivy::schema::Field, value: &str) -> Option<Box<dyn Query>> {
let terms: Vec<_> = value
.split(|character: char| !character.is_alphanumeric())
.filter(|term| !term.is_empty())
.map(|term| {
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(field, term),
IndexRecordOption::WithFreqs,
)) as Box<dyn Query>,
)
})
.collect();
(!terms.is_empty()).then(|| Box::new(BooleanQuery::new(terms)) as Box<dyn Query>)
}
fn normalize_text(value: &str) -> String {
src/search/src/search/indexer.rs
+70 -8
View File
@@ -17,13 +17,16 @@ use tantivy::{
collector::{Count, TopDocs},
directory::MmapDirectory,
query::Query,
tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer},
tokenizer::{LowerCaser, NgramTokenizer, SimpleTokenizer, TextAnalyzer},
};
use super::{
IndexingError, SearchError,
query::{SearchOptions, SearchPage, SearchSort},
schema::{MIXED_NGRAM_TOKENIZER, SearchFields, build_schema, fields_from_schema},
schema::{
FILE_NAME_NGRAM_TOKENIZER, PATH_TOKENIZER, SearchFields, TITLE_NGRAM_TOKENIZER,
build_schema, fields_from_schema,
},
};
const INDEX_WRITER_MEMORY_BYTES: usize = 64 * 1024 * 1024;
@@ -47,9 +50,10 @@ pub struct SearchDiagnostics {
}
struct SearchInner {
index: Index,
index_schema: tantivy::schema::Schema,
reader: IndexReader,
writer: Mutex<IndexWriter>,
writer: Mutex<Option<IndexWriter>>,
fields: SearchFields,
commits: AtomicU64,
commit_failures: AtomicU64,
@@ -118,20 +122,32 @@ impl SearchEngine {
}
fn from_index(index: Index, fields: SearchFields) -> Result<Self, SearchError> {
let analyzer = TextAnalyzer::builder(NgramTokenizer::all_ngrams(1, 20)?)
let title_analyzer = TextAnalyzer::builder(NgramTokenizer::all_ngrams(1, 10)?)
.filter(LowerCaser)
.build();
index.tokenizers().register(MIXED_NGRAM_TOKENIZER, analyzer);
let file_name_analyzer = TextAnalyzer::builder(NgramTokenizer::all_ngrams(2, 8)?)
.filter(LowerCaser)
.build();
let path_analyzer = TextAnalyzer::builder(SimpleTokenizer::default())
.filter(LowerCaser)
.build();
index
.tokenizers()
.register(TITLE_NGRAM_TOKENIZER, title_analyzer);
index
.tokenizers()
.register(FILE_NAME_NGRAM_TOKENIZER, file_name_analyzer);
index.tokenizers().register(PATH_TOKENIZER, path_analyzer);
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()?;
let writer = index.writer_with_num_threads(1, INDEX_WRITER_MEMORY_BYTES)?;
Ok(Self {
inner: Arc::new(SearchInner {
index: index.clone(),
index_schema: index.schema(),
reader,
writer: Mutex::new(writer),
writer: Mutex::new(None),
fields,
commits: AtomicU64::new(0),
commit_failures: AtomicU64::new(0),
@@ -154,6 +170,14 @@ impl SearchEngine {
.writer
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if writer.is_none() {
*writer = Some(
self.inner
.index
.writer_with_num_threads(1, INDEX_WRITER_MEMORY_BYTES)?,
);
}
let writer = writer.as_mut().expect("search writer was initialized");
for group in groups {
writer.delete_term(Term::from_field_text(
fields.content_key,
@@ -192,9 +216,20 @@ impl SearchEngine {
pub fn diagnostics(&self) -> SearchDiagnostics {
let last_commit_at = self.inner.last_commit_at.load(AtomicOrdering::Relaxed);
let writer_initialized = self
.inner
.writer
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_some();
let writer_memory_budget_bytes = if writer_initialized {
INDEX_WRITER_MEMORY_BYTES as u64
} else {
0
};
SearchDiagnostics {
documents: self.num_docs(),
writer_memory_budget_bytes: INDEX_WRITER_MEMORY_BYTES as u64,
writer_memory_budget_bytes,
commits: self.inner.commits.load(AtomicOrdering::Relaxed),
commit_failures: self.inner.commit_failures.load(AtomicOrdering::Relaxed),
last_commit_at: (last_commit_at != 0).then_some(last_commit_at),
@@ -438,6 +473,20 @@ mod tests {
assert_eq!(page.hits[0].seen_count, 4);
}
#[test]
fn writer_is_created_only_when_a_document_is_committed() {
let directory = TempDir::new().unwrap();
let engine = SearchEngine::open(directory.path()).unwrap();
assert_eq!(engine.diagnostics().writer_memory_budget_bytes, 0);
index_records(&engine, &[record()]);
assert_eq!(
engine.diagnostics().writer_memory_budget_bytes,
INDEX_WRITER_MEMORY_BYTES as u64
);
}
#[test]
fn file_path_is_searchable() {
let directory = TempDir::new().unwrap();
@@ -446,6 +495,19 @@ mod tests {
assert_eq!(engine.search("ubuntu.iso", 0, 10).unwrap().total, 1);
}
#[test]
fn long_title_and_file_name_substrings_remain_searchable() {
let directory = TempDir::new().unwrap();
let engine = SearchEngine::open(directory.path()).unwrap();
let mut record = record();
record.name = "Documentary.Collection.Remastered".into();
record.files[0].path = "releases/prefix-verylongreleaseidentifier-suffix.iso".into();
index_records(&engine, &[record]);
assert_eq!(engine.search("collection", 0, 10).unwrap().total, 1);
assert_eq!(engine.search("longrelease", 0, 10).unwrap().total, 1);
}
#[test]
fn large_torrents_index_the_largest_2048_files() {
let directory = TempDir::new().unwrap();
src/search/src/search/runtime.rs
+1 -1
View File
@@ -14,7 +14,7 @@ use crate::storage::IndexInventory;
use super::{SearchDiagnostics, SearchEngine, SearchError, SearchOptions, SearchPage};
const INDEX_DOCUMENT_VERSION: u32 = 2;
const INDEX_DOCUMENT_VERSION: u32 = 3;
const MANAGED_DIRECTORY: &str = "search-index";
const GENERATIONS_DIRECTORY: &str = "generations";
const CURRENT_FILE: &str = "CURRENT";
src/search/src/search/schema.rs
+18 -10
View File
@@ -4,7 +4,9 @@ use tantivy::schema::{
FAST, Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions,
};
pub(crate) const MIXED_NGRAM_TOKENIZER: &str = "dht_mixed_ngram";
pub(crate) const TITLE_NGRAM_TOKENIZER: &str = "dht_title_ngram";
pub(crate) const FILE_NAME_NGRAM_TOKENIZER: &str = "dht_file_name_ngram";
pub(crate) const PATH_TOKENIZER: &str = "dht_path";
#[derive(Debug, Clone, Copy)]
pub(crate) struct SearchFields {
@@ -35,19 +37,17 @@ pub(crate) struct SearchFields {
pub(crate) fn build_schema() -> (Schema, SearchFields) {
let mut builder = Schema::builder();
let info_hash = builder.add_text_field("info_hash", STRING | STORED);
let indexed_text = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(MIXED_NGRAM_TOKENIZER)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
);
let name = builder.add_text_field("name", indexed_text.clone());
let title_text = indexed_text(TITLE_NGRAM_TOKENIZER);
let file_name_text = indexed_text(FILE_NAME_NGRAM_TOKENIZER);
let path_text = indexed_text(PATH_TOKENIZER);
let name = builder.add_text_field("name", title_text.clone());
let exact_name = builder.add_text_field("exact_name", STRING);
let display_name = builder.add_text_field("display_name", STORED);
let aliases = builder.add_text_field("aliases", indexed_text.clone());
let aliases = builder.add_text_field("aliases", title_text);
let exact_aliases = builder.add_text_field("exact_aliases", STRING);
let file_names = builder.add_text_field("file_names", indexed_text.clone());
let file_names = builder.add_text_field("file_names", file_name_text);
let exact_file_names = builder.add_text_field("exact_file_names", STRING);
let files_text = builder.add_text_field("files_text", indexed_text);
let files_text = builder.add_text_field("files_text", path_text);
let regex_text = builder.add_text_field("regex_text", STRING);
let extensions = builder.add_text_field("extensions", STRING);
let total_size = builder.add_u64_field("total_size", FAST | STORED);
@@ -91,6 +91,14 @@ pub(crate) fn build_schema() -> (Schema, SearchFields) {
)
}
fn indexed_text(tokenizer: &'static str) -> TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqs),
)
}
pub(crate) fn fields_from_schema(schema: &Schema) -> tantivy::Result<SearchFields> {
Ok(SearchFields {
info_hash: schema.get_field("info_hash")?,