perf(search): compact file name index
This commit is contained in:
chuan
@@ -148,7 +148,7 @@ bun run build
|
||||
|
||||
大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大
|
||||
|
||||
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引并可与普通关键词组合 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
|
||||
搜索索引对标题使用最多 10 字符的有限 N-Gram 文件名使用 2 至 3 字符 Basic 倒排和独立四字符位置字段 长文件名关键词通过短语查询保证连续匹配 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引并可与普通关键词组合 搜索侧不执行正则或任意通配符扫描并在首次写入前延迟创建 Tantivy writer
|
||||
|
||||
每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB
|
||||
|
||||
@@ -175,7 +175,19 @@ cargo run -p dht-search --bin dht-index-inspect -- data/search-index
|
||||
| 全部 stored fields | 0.20 GiB | 1.39% |
|
||||
| 全部 fast fields | 0.01 GiB | 0.09% |
|
||||
|
||||
空间瓶颈是文件名 N-Gram 不是 stored field fast field 热度或可用性字段 `regex_text` 已无搜索调用方可在下次 Schema 更新中删除 `aliases` 当前重复包含代表标题需要停止重复索引 精确文件名是否保留以及文件名 N-Gram 范围需要通过同一数据集对比搜索覆盖率后决定
|
||||
空间瓶颈是文件名 N-Gram 不是 stored field fast field 热度或可用性字段 新 Schema 已删除没有查询调用方的 `regex_text` 并停止在 `aliases` 中重复索引代表标题 不同 infohash 提供的其他标题仍保留为可搜索别名 精确文件名字段暂时保留用于相关性排序
|
||||
|
||||
在 10 万条确定性记录的相同数据集上对比文件名 N-Gram `2..8` 与 `2..4` 后 `file_names` 从 18232514 字节降到 5068096 字节 单字段下降 72.20% 其中词典下降 91.20% 倒排表下降 65.64% 总 Tantivy 索引下降 12.01% 索引吞吐从每秒 8079 文档提高到 9015 文档
|
||||
|
||||
两组常规查询结果数量保持一致 P95 查询多数变化很小 文件路径片段从 0.213 ms 增加到 0.326 ms 简单 `2..4` 会把长关键词拆成多个四字符词项 测试已确认多个文件分别包含零散片段时 Tantivy 候选可能误匹配
|
||||
|
||||
为避免读取 RocksDB 复核候选破坏稳定分页 实验方案将 2 至 3 字符片段保存为无词频基础倒排 将四字符片段保存到独立位置字段 长关键词使用短语查询 多文件零散片段误匹配测试被精确拦截 连续长文件名搜索保持通过
|
||||
|
||||
在相同 10 万条数据上 拆分位置方案的两个文件名字段合计从 18232514 字节降到 5584568 字节 下降 69.37% 总索引下降 11.51% 索引吞吐从每秒 8079 文档提高到 8934 文档 文件路径片段 P95 为 0.500 ms 位置数据为 1240496 字节
|
||||
|
||||
每条 100 个高度重复短文件名的压力数据中 拆分位置方案空间基本持平但索引吞吐提高约 32% 说明收益取决于真实文件名长度和多样性 按远端字段占比线性推算 相同 Schema 的全新索引可能从约 8.87 GiB 降到约 5.1 GiB
|
||||
|
||||
生产索引已经切换到拆分位置方案 Schema 或文档格式变化时通过影子索引从 RocksDB 全量重建 旧索引在重建完成和原子切换前继续提供搜索 最终体积峰值磁盘和真实查询延迟仍需通过远端重建验证
|
||||
|
||||
## 相关文档
|
||||
|
||||
|
||||
@@ -22,9 +22,7 @@
|
||||
|
||||
- [ ] 在远端真实数据上验证六小时动态状态分桶对 Tantivy 文档重写和删除比例的改善
|
||||
- [ ] 对比优化前后的每小时索引增长提交文档数和段合并压力
|
||||
- [ ] 在下次 Schema 更新中删除已无查询调用方且占真实索引 5.62% 的 `regex_text`
|
||||
- [ ] 停止在 `aliases` 中重复索引代表标题并验证不同 infohash 的真实别名仍可搜索
|
||||
- [ ] 使用同一数据集比较文件名 N-Gram `2..8` `2..4` `2..3` 和 Basic postings 的空间查询延迟与误匹配
|
||||
- [ ] 在远端真实数据的影子重建中验证 2 至 3 字符 Basic 倒排和四字符位置短语拆分方案
|
||||
- [ ] 评估删除 `exact_file_names` 对精确文件名相关性排序的影响
|
||||
- [ ] 评估大型种子文件采样数量和路径文本预算对搜索覆盖率与索引体积的影响
|
||||
- [ ] 明确优化后的单文档平均占用重建峰值空间和预期最大可容纳种子数量
|
||||
@@ -46,7 +44,7 @@
|
||||
|
||||
- [ ] 设计可在查询时计算热度的索引字段避免权重和等级调整触发再次重建
|
||||
- [ ] 为完全同分的搜索结果增加稳定的最终排序键
|
||||
- [ ] 将热度字段稳定排序键和索引空间优化合并为一次 Tantivy Schema 更新
|
||||
- [ ] 优先复用现有字段实现动态热度和稳定排序 若确需新增字段则合并为下一次 Tantivy Schema 更新
|
||||
- [ ] 统计按需验证的 Peer 发现率握手成功率和平均验证耗时
|
||||
- [ ] 根据真实验证数据校准热度权重等级边界和失败退避时间
|
||||
|
||||
@@ -98,8 +96,8 @@
|
||||
|
||||
## 当前下一步
|
||||
|
||||
先根据真实字段占用使用同一数据集比较文件名 N-Gram 候选方案并确定精简 Schema
|
||||
部署紧凑文件名 Schema 并观察远端影子索引的重建进度峰值磁盘占用最终体积和常用查询延迟
|
||||
|
||||
同时继续观察六小时动态状态分桶 使用诊断中的安排刷新抑制刷新实际写入和跳过写入数量确认效果
|
||||
|
||||
Schema 调整时同时加入动态热度所需字段和稳定排序键 最终只执行一次影子索引重建
|
||||
本次 Schema 更新已经删除停用字段和重复代表标题 后续动态热度字段与稳定排序键需要评估是否能够避免再次改变 Schema
|
||||
|
||||
@@ -334,6 +334,9 @@ cargo run --release -p dht-search --bin dht-benchmark -- `
|
||||
| `--query-iterations` | `50` | 每类查询正式采样次数 |
|
||||
| `--query-warmup` | `5` | 每类查询预热次数 |
|
||||
| `--duplicate-every` | `10` | 每多少条创建一个相同内容的不同 infohash 零表示禁用 |
|
||||
| `--file-name-ngram-max` | `8` | 实验文件名 N-Gram 最大长度 只支持 4 或 8 |
|
||||
| `--file-name-ngram-positions` | 不启用 | 为 `2..4` 实验记录位置并使用长关键词短语查询 |
|
||||
| `--file-name-ngram-split` | 不启用 | 将 2 至 3 字符 Basic 倒排与四字符位置字段拆分且必须同时启用位置索引 |
|
||||
| `--output-dir` | `benchmark-data` | 独立运行数据和 JSON 报告根目录 |
|
||||
| `--cleanup` | 不启用 | 报告写入后删除本次 RocksDB 和 Tantivy 数据 |
|
||||
|
||||
|
||||
@@ -23,6 +23,12 @@ pub(crate) struct Args {
|
||||
pub(crate) query_warmup: usize,
|
||||
#[arg(long, default_value_t = 10)]
|
||||
pub(crate) duplicate_every: usize,
|
||||
#[arg(long, default_value_t = 8)]
|
||||
pub(crate) file_name_ngram_max: usize,
|
||||
#[arg(long)]
|
||||
pub(crate) file_name_ngram_positions: bool,
|
||||
#[arg(long)]
|
||||
pub(crate) file_name_ngram_split: bool,
|
||||
#[arg(long)]
|
||||
pub(crate) files_per_record: Option<usize>,
|
||||
#[arg(long, default_value = "benchmark-data")]
|
||||
@@ -46,6 +52,15 @@ impl Args {
|
||||
if self.duplicate_every == 1 {
|
||||
return Err("duplicate-every 必须是零或至少为二".into());
|
||||
}
|
||||
if !matches!(self.file_name_ngram_max, 4 | 8) {
|
||||
return Err("file-name-ngram-max 只支持 4 或 8".into());
|
||||
}
|
||||
if self.file_name_ngram_positions && self.file_name_ngram_max != 4 {
|
||||
return Err("file-name-ngram-positions 只能与最大长度 4 一起使用".into());
|
||||
}
|
||||
if self.file_name_ngram_split && !self.file_name_ngram_positions {
|
||||
return Err("file-name-ngram-split 必须同时启用位置索引".into());
|
||||
}
|
||||
if self
|
||||
.files_per_record
|
||||
.is_some_and(|files| !(1..=20_000).contains(&files))
|
||||
@@ -73,4 +88,27 @@ mod tests {
|
||||
let invalid = Args::parse_from(["benchmark", "--files-per-record", "20001"]);
|
||||
assert!(invalid.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_file_name_ngram_profile() {
|
||||
let compact = Args::parse_from(["benchmark", "--file-name-ngram-max", "4"]);
|
||||
assert!(compact.validate().is_ok());
|
||||
let invalid = Args::parse_from(["benchmark", "--file-name-ngram-max", "5"]);
|
||||
assert!(invalid.validate().is_err());
|
||||
let positioned = Args::parse_from([
|
||||
"benchmark",
|
||||
"--file-name-ngram-max",
|
||||
"4",
|
||||
"--file-name-ngram-positions",
|
||||
]);
|
||||
assert!(positioned.validate().is_ok());
|
||||
let split = Args::parse_from([
|
||||
"benchmark",
|
||||
"--file-name-ngram-max",
|
||||
"4",
|
||||
"--file-name-ngram-positions",
|
||||
"--file-name-ngram-split",
|
||||
]);
|
||||
assert!(split.validate().is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,9 @@ pub(crate) struct BenchmarkReport {
|
||||
pub(crate) records: usize,
|
||||
pub(crate) indexed_documents: u64,
|
||||
pub(crate) duplicate_every: usize,
|
||||
pub(crate) file_name_ngram_max: usize,
|
||||
pub(crate) file_name_ngram_positions: bool,
|
||||
pub(crate) file_name_ngram_split: bool,
|
||||
pub(crate) files_per_record: Option<usize>,
|
||||
pub(crate) generation_seconds: f64,
|
||||
pub(crate) rocksdb_write_seconds: f64,
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::{
|
||||
};
|
||||
|
||||
use dht_search::{
|
||||
search::SearchEngine,
|
||||
search::{SearchEngine, SearchIndexProfile},
|
||||
storage::{RocksTorrentRepository, TorrentRepository},
|
||||
};
|
||||
|
||||
@@ -43,6 +43,9 @@ pub(crate) fn run(args: Args) -> Result<(), Box<dyn Error>> {
|
||||
println!("警告: debug 模式仅用于流程验证 性能结论必须使用 --release");
|
||||
}
|
||||
println!("数据量: {}", format_integer(args.records as u64));
|
||||
println!("文件名 N-Gram: 2..{}", args.file_name_ngram_max);
|
||||
println!("文件名位置索引: {}", args.file_name_ngram_positions);
|
||||
println!("文件名字段拆分: {}", args.file_name_ngram_split);
|
||||
if let Some(files) = args.files_per_record {
|
||||
println!("每条文件数: {}", format_integer(files as u64));
|
||||
}
|
||||
@@ -53,7 +56,17 @@ pub(crate) fn run(args: Args) -> Result<(), Box<dyn Error>> {
|
||||
let repository = RocksTorrentRepository::open(&rocksdb_dir)?;
|
||||
let (generation_duration, write_duration) = populate(&repository, &args)?;
|
||||
|
||||
let search = SearchEngine::open(&tantivy_dir)?;
|
||||
let profile = match (
|
||||
args.file_name_ngram_max,
|
||||
args.file_name_ngram_positions,
|
||||
args.file_name_ngram_split,
|
||||
) {
|
||||
(4, true, true) => SearchIndexProfile::CompactFileNamesSplitExact,
|
||||
(4, true, false) => SearchIndexProfile::CompactFileNamesExact,
|
||||
(4, false, false) => SearchIndexProfile::CompactFileNames,
|
||||
_ => SearchIndexProfile::Standard,
|
||||
};
|
||||
let search = SearchEngine::open_with_profile(&tantivy_dir, profile)?;
|
||||
let index_result = build_index(&repository, &search, &args)?;
|
||||
let mut query_reports = Vec::new();
|
||||
for case in query_cases(args.records, args.duplicate_every)? {
|
||||
@@ -76,6 +89,9 @@ pub(crate) fn run(args: Args) -> Result<(), Box<dyn Error>> {
|
||||
records: args.records,
|
||||
indexed_documents,
|
||||
duplicate_every: args.duplicate_every,
|
||||
file_name_ngram_max: args.file_name_ngram_max,
|
||||
file_name_ngram_positions: args.file_name_ngram_positions,
|
||||
file_name_ngram_split: args.file_name_ngram_split,
|
||||
files_per_record: args.files_per_record,
|
||||
generation_seconds: generation_duration.as_secs_f64(),
|
||||
rocksdb_write_seconds: write_duration.as_secs_f64(),
|
||||
|
||||
@@ -25,15 +25,16 @@ pub(crate) fn from_group(group: &ContentGroup, fields: SearchFields) -> TantivyD
|
||||
if let Some(field) = fields.exact_name {
|
||||
document.add_text(field, &normalized_name);
|
||||
}
|
||||
document.add_text(fields.regex_text, &normalized_name);
|
||||
document.add_text(fields.display_name, &record.name);
|
||||
for alias in &group.aliases {
|
||||
let alias = normalize_bounded(alias, 512);
|
||||
if alias == normalized_name {
|
||||
continue;
|
||||
}
|
||||
document.add_text(fields.aliases, &alias);
|
||||
if let Some(field) = fields.exact_aliases {
|
||||
document.add_text(field, &alias);
|
||||
}
|
||||
document.add_text(fields.regex_text, alias);
|
||||
}
|
||||
let mut indexed_path_bytes = 0_usize;
|
||||
let mut files: Vec<_> = record
|
||||
@@ -60,11 +61,13 @@ pub(crate) fn from_group(group: &ContentGroup, fields: SearchFields) -> TantivyD
|
||||
if let Some(field) = fields.file_names {
|
||||
document.add_text(field, file_name);
|
||||
}
|
||||
if let Some(field) = fields.file_name_quads {
|
||||
document.add_text(field, file_name);
|
||||
}
|
||||
if let Some(field) = fields.exact_file_names {
|
||||
document.add_text(field, file_name);
|
||||
}
|
||||
document.add_text(fields.files_text, &path);
|
||||
document.add_text(fields.regex_text, &path);
|
||||
indexed_files.push(file);
|
||||
}
|
||||
for extension in extensions(&indexed_files) {
|
||||
@@ -183,3 +186,33 @@ fn number(
|
||||
.and_then(|value| value.as_u64())
|
||||
.ok_or(SearchError::MissingField(name))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::domain::{ContentGroupBuilder, InfoHash, test_record};
|
||||
|
||||
use super::*;
|
||||
use crate::search::{SearchIndexProfile, schema::build_schema};
|
||||
|
||||
#[test]
|
||||
fn representative_title_is_not_duplicated_in_alias_fields() {
|
||||
let mut first = test_record(1, 10);
|
||||
first.name = "Representative title".into();
|
||||
let mut second = first.clone();
|
||||
second.info_hash = InfoHash::from_bytes([2; 20]);
|
||||
second.name = "Alternate title".into();
|
||||
let mut builder = ContentGroupBuilder::new(first.content_key, 20);
|
||||
builder.push(first);
|
||||
builder.push(second);
|
||||
let group = builder.finish().unwrap();
|
||||
let (_, fields) = build_schema(SearchIndexProfile::CompactFileNamesSplitExact);
|
||||
|
||||
let document = from_group(&group, fields);
|
||||
let aliases: Vec<_> = document
|
||||
.get_all(fields.aliases)
|
||||
.filter_map(|value| value.as_str())
|
||||
.collect();
|
||||
|
||||
assert_eq!(aliases, vec!["alternate title"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,13 @@ use std::ops::Bound;
|
||||
|
||||
use tantivy::{
|
||||
Term,
|
||||
query::{AllQuery, BooleanQuery, BoostQuery, Occur, Query, RangeQuery, TermQuery},
|
||||
query::{AllQuery, BooleanQuery, BoostQuery, Occur, PhraseQuery, Query, RangeQuery, TermQuery},
|
||||
schema::IndexRecordOption,
|
||||
};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use super::{
|
||||
SearchError,
|
||||
SearchError, SearchIndexProfile,
|
||||
document::availability_number,
|
||||
parse_search_query,
|
||||
query::{SearchOptions, SearchSort},
|
||||
@@ -25,6 +25,7 @@ pub(crate) struct PreparedQuery {
|
||||
pub(crate) fn prepare(
|
||||
options: &SearchOptions,
|
||||
fields: SearchFields,
|
||||
profile: SearchIndexProfile,
|
||||
) -> Result<PreparedQuery, SearchError> {
|
||||
let mut clauses: Vec<Box<dyn Query>> = Vec::new();
|
||||
let query_text = options.query.trim();
|
||||
@@ -33,7 +34,7 @@ pub(crate) fn prepare(
|
||||
clauses.push(Box::new(AllQuery));
|
||||
}
|
||||
if !parsed.text.is_empty() {
|
||||
clauses.push(text_query(&parsed.text, fields)?);
|
||||
clauses.push(text_query(&parsed.text, fields, profile)?);
|
||||
}
|
||||
if let Some(extension) = parsed.extension {
|
||||
clauses.push(Box::new(TermQuery::new(
|
||||
@@ -113,7 +114,11 @@ pub(crate) fn prepare(
|
||||
Ok(PreparedQuery { query, sort })
|
||||
}
|
||||
|
||||
fn text_query(query: &str, fields: SearchFields) -> Result<Box<dyn Query>, SearchError> {
|
||||
fn text_query(
|
||||
query: &str,
|
||||
fields: SearchFields,
|
||||
profile: SearchIndexProfile,
|
||||
) -> Result<Box<dyn Query>, SearchError> {
|
||||
let normalized = normalize_text(query.trim());
|
||||
if normalized.len() == 40 && normalized.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
return Ok(Box::new(TermQuery::new(
|
||||
@@ -168,7 +173,13 @@ fn text_query(query: &str, fields: SearchFields) -> Result<Box<dyn Query>, Searc
|
||||
push_boosted_ngram(&mut alternatives, fields.name, &part, 1, 10, 4.0);
|
||||
push_boosted_ngram(&mut alternatives, fields.aliases, &part, 1, 10, 2.0);
|
||||
if let Some(field) = fields.file_names {
|
||||
push_boosted_ngram(&mut alternatives, field, &part, 2, 8, 1.5);
|
||||
push_file_name_query(
|
||||
&mut alternatives,
|
||||
field,
|
||||
fields.file_name_quads,
|
||||
&part,
|
||||
profile,
|
||||
);
|
||||
}
|
||||
if let Some(query) = path_query(fields.files_text, &part) {
|
||||
alternatives.push((Occur::Should, query));
|
||||
@@ -184,6 +195,71 @@ fn text_query(query: &str, fields: SearchFields) -> Result<Box<dyn Query>, Searc
|
||||
Ok(Box::new(BooleanQuery::new(required)))
|
||||
}
|
||||
|
||||
fn push_file_name_query(
|
||||
alternatives: &mut Vec<(Occur, Box<dyn Query>)>,
|
||||
field: tantivy::schema::Field,
|
||||
quad_field: Option<tantivy::schema::Field>,
|
||||
value: &str,
|
||||
profile: SearchIndexProfile,
|
||||
) {
|
||||
let characters: Vec<_> = value.chars().collect();
|
||||
if profile.split_file_name_fields() {
|
||||
if characters.len() < 2 {
|
||||
return;
|
||||
}
|
||||
if characters.len() <= 3 {
|
||||
alternatives.push((
|
||||
Occur::Should,
|
||||
Box::new(BoostQuery::new(
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(field, value),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
1.5,
|
||||
)),
|
||||
));
|
||||
return;
|
||||
}
|
||||
let quad_field = quad_field.expect("split file name profile has a quad field");
|
||||
let terms: Vec<_> = characters
|
||||
.windows(4)
|
||||
.map(|window| {
|
||||
let chunk: String = window.iter().collect();
|
||||
Term::from_field_text(quad_field, &chunk)
|
||||
})
|
||||
.collect();
|
||||
let query: Box<dyn Query> = if terms.len() == 1 {
|
||||
Box::new(TermQuery::new(terms[0].clone(), IndexRecordOption::Basic))
|
||||
} else {
|
||||
Box::new(PhraseQuery::new(terms))
|
||||
};
|
||||
alternatives.push((Occur::Should, Box::new(BoostQuery::new(query, 1.5))));
|
||||
return;
|
||||
}
|
||||
if profile.exact_file_name_phrases() && characters.len() > 4 {
|
||||
let terms = characters
|
||||
.windows(4)
|
||||
.map(|window| {
|
||||
let chunk: String = window.iter().collect();
|
||||
Term::from_field_text(field, &chunk)
|
||||
})
|
||||
.collect();
|
||||
alternatives.push((
|
||||
Occur::Should,
|
||||
Box::new(BoostQuery::new(Box::new(PhraseQuery::new(terms)), 1.5)),
|
||||
));
|
||||
return;
|
||||
}
|
||||
push_boosted_ngram(
|
||||
alternatives,
|
||||
field,
|
||||
value,
|
||||
2,
|
||||
profile.file_name_max_gram(),
|
||||
1.5,
|
||||
);
|
||||
}
|
||||
|
||||
fn push_boosted_ngram(
|
||||
alternatives: &mut Vec<(Occur, Box<dyn Query>)>,
|
||||
field: tantivy::schema::Field,
|
||||
|
||||
@@ -25,14 +25,56 @@ use super::{
|
||||
IndexingError, SearchError,
|
||||
query::{SearchOptions, SearchPage, SearchSort},
|
||||
schema::{
|
||||
FILE_NAME_NGRAM_TOKENIZER, PATH_TOKENIZER, SearchFields, TITLE_NGRAM_TOKENIZER,
|
||||
build_schema, fields_from_schema,
|
||||
COMPACT_FILE_NAME_NGRAM_TOKENIZER, FILE_NAME_NGRAM_TOKENIZER, PATH_TOKENIZER,
|
||||
POSITIONED_FILE_NAME_NGRAM_TOKENIZER, SHORT_FILE_NAME_NGRAM_TOKENIZER, SearchFields,
|
||||
TITLE_NGRAM_TOKENIZER, build_schema, fields_from_schema,
|
||||
},
|
||||
tokenizer::PositionedNgramTokenizer,
|
||||
};
|
||||
|
||||
const INDEX_WRITER_MEMORY_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_PAGE_SIZE: usize = 100;
|
||||
const MAX_OFFSET: usize = 10_000;
|
||||
const PRODUCTION_PROFILE: SearchIndexProfile = SearchIndexProfile::CompactFileNamesSplitExact;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum SearchIndexProfile {
|
||||
#[default]
|
||||
Standard,
|
||||
CompactFileNames,
|
||||
CompactFileNamesExact,
|
||||
CompactFileNamesSplitExact,
|
||||
}
|
||||
|
||||
impl SearchIndexProfile {
|
||||
pub(crate) fn file_name_max_gram(self) -> usize {
|
||||
match self {
|
||||
Self::Standard => 8,
|
||||
Self::CompactFileNames | Self::CompactFileNamesExact => 4,
|
||||
Self::CompactFileNamesSplitExact => 3,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn file_name_tokenizer(self) -> &'static str {
|
||||
match self {
|
||||
Self::Standard => FILE_NAME_NGRAM_TOKENIZER,
|
||||
Self::CompactFileNames => COMPACT_FILE_NAME_NGRAM_TOKENIZER,
|
||||
Self::CompactFileNamesExact => POSITIONED_FILE_NAME_NGRAM_TOKENIZER,
|
||||
Self::CompactFileNamesSplitExact => SHORT_FILE_NAME_NGRAM_TOKENIZER,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn exact_file_name_phrases(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::CompactFileNamesExact | Self::CompactFileNamesSplitExact
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn split_file_name_fields(self) -> bool {
|
||||
self == Self::CompactFileNamesSplitExact
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SearchEngine {
|
||||
@@ -58,6 +100,7 @@ struct SearchInner {
|
||||
reader: IndexReader,
|
||||
writer: Mutex<Option<IndexWriter>>,
|
||||
fields: SearchFields,
|
||||
profile: SearchIndexProfile,
|
||||
commits: AtomicU64,
|
||||
commit_failures: AtomicU64,
|
||||
last_commit_at: AtomicU64,
|
||||
@@ -72,6 +115,13 @@ impl SearchEngine {
|
||||
Self::open_with_status(path).map(|(engine, _)| engine)
|
||||
}
|
||||
|
||||
pub fn open_with_profile(
|
||||
path: impl AsRef<Path>,
|
||||
profile: SearchIndexProfile,
|
||||
) -> Result<Self, SearchError> {
|
||||
Self::open_with_status_and_profile(path, profile).map(|(engine, _)| engine)
|
||||
}
|
||||
|
||||
pub fn recreate(path: impl AsRef<Path>) -> Result<Self, SearchError> {
|
||||
let path = path.as_ref();
|
||||
if path.exists() {
|
||||
@@ -82,12 +132,19 @@ impl SearchEngine {
|
||||
}
|
||||
|
||||
pub fn open_with_status(path: impl AsRef<Path>) -> Result<(Self, bool), SearchError> {
|
||||
Self::open_with_status_and_profile(path, PRODUCTION_PROFILE)
|
||||
}
|
||||
|
||||
fn open_with_status_and_profile(
|
||||
path: impl AsRef<Path>,
|
||||
profile: SearchIndexProfile,
|
||||
) -> Result<(Self, bool), SearchError> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
std::fs::create_dir_all(&path)
|
||||
.map_err(|error| SearchError::Directory(error.to_string()))?;
|
||||
let mut directory = MmapDirectory::open(&path)
|
||||
.map_err(|error| SearchError::Directory(error.to_string()))?;
|
||||
let (expected_schema, _) = build_schema();
|
||||
let (expected_schema, _) = build_schema(profile);
|
||||
let exists =
|
||||
Index::exists(&directory).map_err(|error| SearchError::Directory(error.to_string()))?;
|
||||
let mut created = !exists;
|
||||
@@ -110,7 +167,7 @@ impl SearchEngine {
|
||||
Index::open_or_create(directory, expected_schema)?
|
||||
};
|
||||
let fields = fields_from_schema(&index.schema())?;
|
||||
Ok((Self::from_index(index, fields)?, created))
|
||||
Ok((Self::from_index(index, fields, profile)?, created))
|
||||
}
|
||||
|
||||
pub(crate) fn open_existing(path: impl AsRef<Path>) -> Result<Self, SearchError> {
|
||||
@@ -119,20 +176,43 @@ impl SearchEngine {
|
||||
MmapDirectory::open(path).map_err(|error| SearchError::Directory(error.to_string()))?;
|
||||
let index = Index::open(directory)?;
|
||||
let fields = fields_from_schema(&index.schema())?;
|
||||
Self::from_index(index, fields)
|
||||
let profile = if fields.file_name_quads.is_some() {
|
||||
SearchIndexProfile::CompactFileNamesSplitExact
|
||||
} else {
|
||||
SearchIndexProfile::Standard
|
||||
};
|
||||
Self::from_index(index, fields, profile)
|
||||
}
|
||||
|
||||
pub(crate) fn has_current_schema(&self) -> bool {
|
||||
self.inner.index_schema == build_schema().0
|
||||
self.inner.index_schema == build_schema(PRODUCTION_PROFILE).0
|
||||
}
|
||||
|
||||
fn from_index(index: Index, fields: SearchFields) -> Result<Self, SearchError> {
|
||||
fn from_index(
|
||||
index: Index,
|
||||
fields: SearchFields,
|
||||
profile: SearchIndexProfile,
|
||||
) -> Result<Self, SearchError> {
|
||||
let title_analyzer = TextAnalyzer::builder(NgramTokenizer::all_ngrams(1, 10)?)
|
||||
.filter(LowerCaser)
|
||||
.build();
|
||||
let file_name_analyzer = TextAnalyzer::builder(NgramTokenizer::all_ngrams(2, 8)?)
|
||||
.filter(LowerCaser)
|
||||
.build();
|
||||
let file_name_analyzer = match profile {
|
||||
SearchIndexProfile::CompactFileNamesExact => {
|
||||
TextAnalyzer::builder(PositionedNgramTokenizer::new(2, 4)?)
|
||||
.filter(LowerCaser)
|
||||
.build()
|
||||
}
|
||||
SearchIndexProfile::CompactFileNamesSplitExact => {
|
||||
TextAnalyzer::builder(NgramTokenizer::all_ngrams(2, 3)?)
|
||||
.filter(LowerCaser)
|
||||
.build()
|
||||
}
|
||||
_ => {
|
||||
TextAnalyzer::builder(NgramTokenizer::all_ngrams(2, profile.file_name_max_gram())?)
|
||||
.filter(LowerCaser)
|
||||
.build()
|
||||
}
|
||||
};
|
||||
let path_analyzer = TextAnalyzer::builder(SimpleTokenizer::default())
|
||||
.filter(LowerCaser)
|
||||
.build();
|
||||
@@ -141,7 +221,15 @@ impl SearchEngine {
|
||||
.register(TITLE_NGRAM_TOKENIZER, title_analyzer);
|
||||
index
|
||||
.tokenizers()
|
||||
.register(FILE_NAME_NGRAM_TOKENIZER, file_name_analyzer);
|
||||
.register(profile.file_name_tokenizer(), file_name_analyzer);
|
||||
if profile.split_file_name_fields() {
|
||||
let quad_analyzer = TextAnalyzer::builder(PositionedNgramTokenizer::new(4, 4)?)
|
||||
.filter(LowerCaser)
|
||||
.build();
|
||||
index
|
||||
.tokenizers()
|
||||
.register(POSITIONED_FILE_NAME_NGRAM_TOKENIZER, quad_analyzer);
|
||||
}
|
||||
index.tokenizers().register(PATH_TOKENIZER, path_analyzer);
|
||||
let reader = index
|
||||
.reader_builder()
|
||||
@@ -154,6 +242,7 @@ impl SearchEngine {
|
||||
reader,
|
||||
writer: Mutex::new(None),
|
||||
fields,
|
||||
profile,
|
||||
commits: AtomicU64::new(0),
|
||||
commit_failures: AtomicU64::new(0),
|
||||
last_commit_at: AtomicU64::new(0),
|
||||
@@ -333,7 +422,7 @@ impl SearchEngine {
|
||||
let offset = options.offset.min(MAX_OFFSET);
|
||||
let limit = options.limit.clamp(1, MAX_PAGE_SIZE);
|
||||
let fields = self.inner.fields;
|
||||
let prepared = super::filter::prepare(&options, fields)?;
|
||||
let prepared = super::filter::prepare(&options, fields, self.inner.profile)?;
|
||||
let query = prepared.query;
|
||||
let searcher = self.inner.reader.searcher();
|
||||
let sort = prepared.sort;
|
||||
@@ -581,6 +670,80 @@ mod tests {
|
||||
assert_eq!(engine.search("longrelease", 0, 10).unwrap().total, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_file_ngrams_can_return_cross_file_fragment_candidates() {
|
||||
let standard_directory = TempDir::new().unwrap();
|
||||
let compact_directory = TempDir::new().unwrap();
|
||||
let exact_directory = TempDir::new().unwrap();
|
||||
let split_directory = TempDir::new().unwrap();
|
||||
let standard = SearchEngine::open_with_profile(
|
||||
standard_directory.path(),
|
||||
SearchIndexProfile::Standard,
|
||||
)
|
||||
.unwrap();
|
||||
let compact = SearchEngine::open_with_profile(
|
||||
compact_directory.path(),
|
||||
SearchIndexProfile::CompactFileNames,
|
||||
)
|
||||
.unwrap();
|
||||
let exact = SearchEngine::open_with_profile(
|
||||
exact_directory.path(),
|
||||
SearchIndexProfile::CompactFileNamesExact,
|
||||
)
|
||||
.unwrap();
|
||||
let split = SearchEngine::open_with_profile(
|
||||
split_directory.path(),
|
||||
SearchIndexProfile::CompactFileNamesSplitExact,
|
||||
)
|
||||
.unwrap();
|
||||
let mut record = record();
|
||||
record.name = "Unrelated collection".into();
|
||||
record.files = vec![
|
||||
TorrentFile {
|
||||
path: "parts/abcd.txt".into(),
|
||||
size: 1,
|
||||
},
|
||||
TorrentFile {
|
||||
path: "parts/bcde.txt".into(),
|
||||
size: 1,
|
||||
},
|
||||
TorrentFile {
|
||||
path: "parts/cdef.txt".into(),
|
||||
size: 1,
|
||||
},
|
||||
];
|
||||
record.total_size = 3;
|
||||
record.content_key = crate::domain::content_key(&record.files).unwrap();
|
||||
index_records(&standard, std::slice::from_ref(&record));
|
||||
index_records(&compact, std::slice::from_ref(&record));
|
||||
index_records(&exact, std::slice::from_ref(&record));
|
||||
index_records(&split, &[record]);
|
||||
|
||||
assert_eq!(standard.search("abcdef", 0, 10).unwrap().total, 0);
|
||||
assert_eq!(compact.search("abcdef", 0, 10).unwrap().total, 1);
|
||||
assert_eq!(exact.search("abcdef", 0, 10).unwrap().total, 0);
|
||||
assert_eq!(split.search("abcdef", 0, 10).unwrap().total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positioned_compact_ngrams_match_a_contiguous_long_file_name() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let engine = SearchEngine::open_with_profile(
|
||||
directory.path(),
|
||||
SearchIndexProfile::CompactFileNamesSplitExact,
|
||||
)
|
||||
.unwrap();
|
||||
let mut record = record();
|
||||
record.name = "Unrelated collection".into();
|
||||
record.files[0].path = "images/ubuntu-desktop.iso".into();
|
||||
record.content_key = crate::domain::content_key(&record.files).unwrap();
|
||||
index_records(&engine, &[record]);
|
||||
|
||||
assert_eq!(engine.search("ubuntu", 0, 10).unwrap().total, 1);
|
||||
assert_eq!(engine.search("desktop", 0, 10).unwrap().total, 1);
|
||||
assert_eq!(engine.search("ubuntu-desktop", 0, 10).unwrap().total, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_torrents_index_the_largest_2048_files() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
|
||||
@@ -7,9 +7,10 @@ mod indexer;
|
||||
mod query;
|
||||
mod runtime;
|
||||
mod schema;
|
||||
mod tokenizer;
|
||||
|
||||
pub(crate) use file_order::order_files;
|
||||
pub use indexer::{SearchDiagnostics, SearchEngine};
|
||||
pub use indexer::{SearchDiagnostics, SearchEngine, SearchIndexProfile};
|
||||
pub use query::{AvailabilitySummary, SearchHit, SearchOptions, SearchPage, SearchSort};
|
||||
pub use runtime::{IndexRebuildReason, IndexState, IndexStatus, SearchRuntime};
|
||||
pub(crate) use runtime::{RebuildSession, SearchBootstrap};
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::storage::IndexInventory;
|
||||
|
||||
use super::{SearchDiagnostics, SearchEngine, SearchError, SearchOptions, SearchPage};
|
||||
|
||||
const INDEX_DOCUMENT_VERSION: u32 = 3;
|
||||
const INDEX_DOCUMENT_VERSION: u32 = 4;
|
||||
const MANAGED_DIRECTORY: &str = "search-index";
|
||||
const GENERATIONS_DIRECTORY: &str = "generations";
|
||||
const CURRENT_FILE: &str = "CURRENT";
|
||||
|
||||
@@ -4,8 +4,13 @@ use tantivy::schema::{
|
||||
FAST, Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions,
|
||||
};
|
||||
|
||||
use super::SearchIndexProfile;
|
||||
|
||||
pub(crate) const TITLE_NGRAM_TOKENIZER: &str = "dht_title_ngram";
|
||||
pub(crate) const FILE_NAME_NGRAM_TOKENIZER: &str = "dht_file_name_ngram";
|
||||
pub(crate) const COMPACT_FILE_NAME_NGRAM_TOKENIZER: &str = "dht_file_name_ngram_2_4";
|
||||
pub(crate) const POSITIONED_FILE_NAME_NGRAM_TOKENIZER: &str = "dht_file_name_ngram_2_4_positioned";
|
||||
pub(crate) const SHORT_FILE_NAME_NGRAM_TOKENIZER: &str = "dht_file_name_ngram_2_3_basic";
|
||||
pub(crate) const PATH_TOKENIZER: &str = "dht_path";
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -17,9 +22,9 @@ pub(crate) struct SearchFields {
|
||||
pub(crate) aliases: Field,
|
||||
pub(crate) exact_aliases: Option<Field>,
|
||||
pub(crate) file_names: Option<Field>,
|
||||
pub(crate) file_name_quads: Option<Field>,
|
||||
pub(crate) exact_file_names: Option<Field>,
|
||||
pub(crate) files_text: Field,
|
||||
pub(crate) regex_text: Field,
|
||||
pub(crate) extensions: Field,
|
||||
pub(crate) total_size: Field,
|
||||
pub(crate) file_count: Field,
|
||||
@@ -34,11 +39,26 @@ pub(crate) struct SearchFields {
|
||||
pub(crate) variant_count: Field,
|
||||
}
|
||||
|
||||
pub(crate) fn build_schema() -> (Schema, SearchFields) {
|
||||
pub(crate) fn build_schema(profile: SearchIndexProfile) -> (Schema, SearchFields) {
|
||||
let mut builder = Schema::builder();
|
||||
let info_hash = builder.add_text_field("info_hash", STRING | STORED);
|
||||
let title_text = indexed_text(TITLE_NGRAM_TOKENIZER);
|
||||
let file_name_text = indexed_text(FILE_NAME_NGRAM_TOKENIZER);
|
||||
let file_name_text = if profile.split_file_name_fields() {
|
||||
indexed_text_with_options(
|
||||
profile.file_name_tokenizer(),
|
||||
IndexRecordOption::Basic,
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
indexed_text_with_option(
|
||||
profile.file_name_tokenizer(),
|
||||
if profile.exact_file_name_phrases() {
|
||||
IndexRecordOption::WithFreqsAndPositions
|
||||
} else {
|
||||
IndexRecordOption::WithFreqs
|
||||
},
|
||||
)
|
||||
};
|
||||
let path_text = indexed_text(PATH_TOKENIZER);
|
||||
let name = builder.add_text_field("name", title_text.clone());
|
||||
let exact_name = builder.add_text_field("exact_name", STRING);
|
||||
@@ -46,9 +66,18 @@ pub(crate) fn build_schema() -> (Schema, SearchFields) {
|
||||
let aliases = builder.add_text_field("aliases", title_text);
|
||||
let exact_aliases = builder.add_text_field("exact_aliases", STRING);
|
||||
let file_names = builder.add_text_field("file_names", file_name_text);
|
||||
let file_name_quads = profile.split_file_name_fields().then(|| {
|
||||
builder.add_text_field(
|
||||
"file_name_quads",
|
||||
indexed_text_with_options(
|
||||
POSITIONED_FILE_NAME_NGRAM_TOKENIZER,
|
||||
IndexRecordOption::WithFreqsAndPositions,
|
||||
false,
|
||||
),
|
||||
)
|
||||
});
|
||||
let exact_file_names = builder.add_text_field("exact_file_names", STRING);
|
||||
let files_text = builder.add_text_field("files_text", path_text);
|
||||
let regex_text = builder.add_text_field("regex_text", STRING);
|
||||
let extensions = builder.add_text_field("extensions", STRING);
|
||||
let total_size = builder.add_u64_field("total_size", FAST | STORED);
|
||||
let file_count = builder.add_u64_field("file_count", FAST | STORED);
|
||||
@@ -72,9 +101,9 @@ pub(crate) fn build_schema() -> (Schema, SearchFields) {
|
||||
aliases,
|
||||
exact_aliases: Some(exact_aliases),
|
||||
file_names: Some(file_names),
|
||||
file_name_quads,
|
||||
exact_file_names: Some(exact_file_names),
|
||||
files_text,
|
||||
regex_text,
|
||||
extensions,
|
||||
total_size,
|
||||
file_count,
|
||||
@@ -92,10 +121,26 @@ pub(crate) fn build_schema() -> (Schema, SearchFields) {
|
||||
}
|
||||
|
||||
fn indexed_text(tokenizer: &'static str) -> TextOptions {
|
||||
indexed_text_with_option(tokenizer, IndexRecordOption::WithFreqs)
|
||||
}
|
||||
|
||||
fn indexed_text_with_option(
|
||||
tokenizer: &'static str,
|
||||
index_option: IndexRecordOption,
|
||||
) -> TextOptions {
|
||||
indexed_text_with_options(tokenizer, index_option, true)
|
||||
}
|
||||
|
||||
fn indexed_text_with_options(
|
||||
tokenizer: &'static str,
|
||||
index_option: IndexRecordOption,
|
||||
fieldnorms: bool,
|
||||
) -> TextOptions {
|
||||
TextOptions::default().set_indexing_options(
|
||||
TextFieldIndexing::default()
|
||||
.set_tokenizer(tokenizer)
|
||||
.set_index_option(IndexRecordOption::WithFreqs),
|
||||
.set_index_option(index_option)
|
||||
.set_fieldnorms(fieldnorms),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,9 +153,9 @@ pub(crate) fn fields_from_schema(schema: &Schema) -> tantivy::Result<SearchField
|
||||
aliases: schema.get_field("aliases")?,
|
||||
exact_aliases: schema.get_field("exact_aliases").ok(),
|
||||
file_names: schema.get_field("file_names").ok(),
|
||||
file_name_quads: schema.get_field("file_name_quads").ok(),
|
||||
exact_file_names: schema.get_field("exact_file_names").ok(),
|
||||
files_text: schema.get_field("files_text")?,
|
||||
regex_text: schema.get_field("regex_text")?,
|
||||
extensions: schema.get_field("extensions")?,
|
||||
total_size: schema.get_field("total_size")?,
|
||||
file_count: schema.get_field("file_count")?,
|
||||
@@ -125,3 +170,17 @@ pub(crate) fn fields_from_schema(schema: &Schema) -> tantivy::Result<SearchField
|
||||
variant_count: schema.get_field("variant_count")?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn production_schema_uses_split_file_name_fields_without_legacy_regex_field() {
|
||||
let (schema, fields) = build_schema(SearchIndexProfile::CompactFileNamesSplitExact);
|
||||
|
||||
assert!(fields.file_name_quads.is_some());
|
||||
assert!(schema.get_field("file_name_quads").is_ok());
|
||||
assert!(schema.get_field("regex_text").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// 负责生成带字符起始位置的 N-Gram 词项供精确短语查询使用
|
||||
|
||||
use tantivy::{
|
||||
TantivyError,
|
||||
tokenizer::{Token, TokenStream, Tokenizer},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct PositionedNgramTokenizer {
|
||||
min_gram: usize,
|
||||
max_gram: usize,
|
||||
boundaries: Vec<usize>,
|
||||
token: Token,
|
||||
}
|
||||
|
||||
impl PositionedNgramTokenizer {
|
||||
pub(crate) fn new(min_gram: usize, max_gram: usize) -> tantivy::Result<Self> {
|
||||
if min_gram == 0 || min_gram > max_gram {
|
||||
return Err(TantivyError::InvalidArgument(
|
||||
"N-Gram 长度范围无效".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
min_gram,
|
||||
max_gram,
|
||||
boundaries: Vec::new(),
|
||||
token: Token::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct PositionedNgramTokenStream<'a> {
|
||||
text: &'a str,
|
||||
boundaries: &'a [usize],
|
||||
min_gram: usize,
|
||||
max_gram: usize,
|
||||
start: usize,
|
||||
gram_len: usize,
|
||||
token: &'a mut Token,
|
||||
}
|
||||
|
||||
impl Tokenizer for PositionedNgramTokenizer {
|
||||
type TokenStream<'a> = PositionedNgramTokenStream<'a>;
|
||||
|
||||
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
|
||||
self.boundaries.clear();
|
||||
self.boundaries
|
||||
.extend(text.char_indices().map(|(offset, _)| offset));
|
||||
self.boundaries.push(text.len());
|
||||
self.token.reset();
|
||||
PositionedNgramTokenStream {
|
||||
text,
|
||||
boundaries: &self.boundaries,
|
||||
min_gram: self.min_gram,
|
||||
max_gram: self.max_gram,
|
||||
start: 0,
|
||||
gram_len: self.min_gram,
|
||||
token: &mut self.token,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenStream for PositionedNgramTokenStream<'_> {
|
||||
fn advance(&mut self) -> bool {
|
||||
loop {
|
||||
if self.start + self.min_gram >= self.boundaries.len() {
|
||||
return false;
|
||||
}
|
||||
if self.gram_len > self.max_gram || self.start + self.gram_len >= self.boundaries.len()
|
||||
{
|
||||
self.start += 1;
|
||||
self.gram_len = self.min_gram;
|
||||
continue;
|
||||
}
|
||||
let offset_from = self.boundaries[self.start];
|
||||
let offset_to = self.boundaries[self.start + self.gram_len];
|
||||
self.token.position = self.start;
|
||||
self.token.position_length = 1;
|
||||
self.token.offset_from = offset_from;
|
||||
self.token.offset_to = offset_to;
|
||||
self.token.text.clear();
|
||||
self.token.text.push_str(&self.text[offset_from..offset_to]);
|
||||
self.gram_len += 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
self.token
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
self.token
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::PositionedNgramTokenizer;
|
||||
use tantivy::tokenizer::{TokenStream, Tokenizer};
|
||||
|
||||
#[test]
|
||||
fn emits_all_lengths_at_their_character_start_position() {
|
||||
let mut tokenizer = PositionedNgramTokenizer::new(2, 4).unwrap();
|
||||
let mut stream = tokenizer.token_stream("测试abc");
|
||||
let mut tokens = Vec::new();
|
||||
stream.process(&mut |token| {
|
||||
tokens.push((token.text.clone(), token.position));
|
||||
});
|
||||
assert!(tokens.contains(&("测试".to_owned(), 0)));
|
||||
assert!(tokens.contains(&("测试ab".to_owned(), 0)));
|
||||
assert!(tokens.contains(&("试abc".to_owned(), 1)));
|
||||
assert!(tokens.contains(&("abc".to_owned(), 2)));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user