feat(search): combine text and extension queries
This commit is contained in:
@@ -148,7 +148,7 @@ bun run build
|
||||
|
||||
大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大
|
||||
|
||||
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
|
||||
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引并可与普通关键词组合 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
|
||||
|
||||
每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
### 验收标准
|
||||
|
||||
- [ ] 使用远端真实数据或等比例数据对比优化前后的索引体积
|
||||
- [ ] 普通文本扩展名快捷查询精确哈希和详情文件匹配测试保持通过
|
||||
- [ ] 普通文本与扩展名组合查询精确哈希和详情文件匹配测试保持通过
|
||||
- [ ] 影子索引重建期间旧索引继续提供搜索且完成后能够原子切换
|
||||
- [ ] 重建峰值空间不会触发磁盘只读保护
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ GET /torrents/{infohash}?file_offset=0&file_limit=100
|
||||
|
||||
搜索支持中文英文数字和文件名片段匹配
|
||||
|
||||
搜索支持不区分大小写的单扩展名快捷语法 例如 `*.iso` 直接匹配包含 ISO 文件的已索引内容 普通关键词继续匹配标题别名文件名和路径片段
|
||||
搜索支持不区分大小写的单扩展名快捷语法 例如 `*.iso` 直接匹配包含 ISO 文件的已索引内容 `Ubuntu *.iso` 表示普通关键词和扩展名条件同时满足 普通关键词继续匹配标题别名文件名和路径片段
|
||||
|
||||
搜索接口不接受正则模式 `mode` 参数 也不接受 `*iso` `ubuntu*` `?` 或多段扩展名等任意通配符表达式
|
||||
|
||||
|
||||
@@ -411,6 +411,22 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(json["total"], 1);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/search?q=Example%20%2A.mkv")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(json["total"], 1);
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
|
||||
@@ -6,7 +6,7 @@ use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::domain::TorrentFile;
|
||||
|
||||
use super::{SearchError, SearchSort, extension_shortcut};
|
||||
use super::{SearchError, SearchSort, parse_search_query};
|
||||
|
||||
pub(crate) fn order_files(
|
||||
files: Vec<TorrentFile>,
|
||||
@@ -42,13 +42,7 @@ impl RankedFile {
|
||||
fn new(file: TorrentFile, matcher: &FileMatcher) -> Self {
|
||||
let path = normalize(&file.path).replace('\\', "/");
|
||||
let name = path.rsplit('/').next().unwrap_or(path.as_str()).to_owned();
|
||||
let relevance = if matcher.is_match(&name) {
|
||||
2
|
||||
} else if matcher.is_match(&path) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let relevance = matcher.relevance(&name, &path);
|
||||
Self {
|
||||
file,
|
||||
name,
|
||||
@@ -58,33 +52,45 @@ impl RankedFile {
|
||||
}
|
||||
}
|
||||
|
||||
enum FileMatcher {
|
||||
None,
|
||||
Plain(Vec<String>),
|
||||
Extension(String),
|
||||
struct FileMatcher {
|
||||
terms: Vec<String>,
|
||||
extension: Option<String>,
|
||||
}
|
||||
|
||||
impl FileMatcher {
|
||||
fn compile(query: &str) -> Result<Self, SearchError> {
|
||||
let query = query.trim();
|
||||
if query.is_empty() {
|
||||
return Ok(Self::None);
|
||||
}
|
||||
if let Some(extension) = extension_shortcut(query)? {
|
||||
return Ok(Self::Extension(format!(".{extension}")));
|
||||
}
|
||||
let terms = normalize(query)
|
||||
let parsed = parse_search_query(query)?;
|
||||
let terms = normalize(&parsed.text)
|
||||
.split_whitespace()
|
||||
.map(ToOwned::to_owned)
|
||||
.collect();
|
||||
Ok(Self::Plain(terms))
|
||||
Ok(Self {
|
||||
terms,
|
||||
extension: parsed.extension.map(|value| format!(".{value}")),
|
||||
})
|
||||
}
|
||||
|
||||
fn is_match(&self, value: &str) -> bool {
|
||||
match self {
|
||||
Self::None => false,
|
||||
Self::Plain(terms) => terms.iter().all(|term| value.contains(term)),
|
||||
Self::Extension(extension) => value.ends_with(extension),
|
||||
fn relevance(&self, name: &str, path: &str) -> u8 {
|
||||
let text_relevance = if self.terms.is_empty() {
|
||||
0
|
||||
} else if self.terms.iter().all(|term| name.contains(term)) {
|
||||
2
|
||||
} else if self.terms.iter().all(|term| path.contains(term)) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let extension_matches = self
|
||||
.extension
|
||||
.as_ref()
|
||||
.is_some_and(|extension| name.ends_with(extension));
|
||||
match (text_relevance, extension_matches) {
|
||||
(2, true) => 5,
|
||||
(1, true) => 4,
|
||||
(2, false) => 3,
|
||||
(1, false) => 2,
|
||||
(0, true) => 1,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,4 +175,19 @@ mod tests {
|
||||
let files = vec![file("movie.iso", 1)];
|
||||
assert!(order_files(files, "*iso", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn combined_query_prefers_files_matching_text_and_extension() {
|
||||
let files = vec![
|
||||
file("ubuntu.txt", 1),
|
||||
file("disk.iso", 2),
|
||||
file("ubuntu.iso", 3),
|
||||
file("other.bin", 4),
|
||||
];
|
||||
let ordered = order_files(files, "Ubuntu *.ISO", None).unwrap();
|
||||
assert_eq!(ordered[0].path, "ubuntu.iso");
|
||||
assert_eq!(ordered[1].path, "ubuntu.txt");
|
||||
assert_eq!(ordered[2].path, "disk.iso");
|
||||
assert_eq!(ordered[3].path, "other.bin");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use unicode_normalization::UnicodeNormalization;
|
||||
use super::{
|
||||
SearchError,
|
||||
document::availability_number,
|
||||
extension_shortcut,
|
||||
parse_search_query,
|
||||
query::{SearchOptions, SearchSort},
|
||||
schema::SearchFields,
|
||||
};
|
||||
@@ -28,15 +28,18 @@ pub(crate) fn prepare(
|
||||
) -> Result<PreparedQuery, SearchError> {
|
||||
let mut clauses: Vec<Box<dyn Query>> = Vec::new();
|
||||
let query_text = options.query.trim();
|
||||
if query_text.is_empty() {
|
||||
let parsed = parse_search_query(query_text)?;
|
||||
if parsed.text.is_empty() && parsed.extension.is_none() {
|
||||
clauses.push(Box::new(AllQuery));
|
||||
} else if let Some(extension) = extension_shortcut(query_text)? {
|
||||
}
|
||||
if !parsed.text.is_empty() {
|
||||
clauses.push(text_query(&parsed.text, fields)?);
|
||||
}
|
||||
if let Some(extension) = parsed.extension {
|
||||
clauses.push(Box::new(TermQuery::new(
|
||||
Term::from_field_text(fields.extensions, &extension),
|
||||
IndexRecordOption::Basic,
|
||||
)));
|
||||
} else {
|
||||
clauses.push(text_query(query_text, fields)?);
|
||||
}
|
||||
if let Some(content_key) = options.content_key {
|
||||
clauses.push(Box::new(TermQuery::new(
|
||||
|
||||
@@ -697,6 +697,31 @@ mod tests {
|
||||
assert_eq!(missing.total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_and_extension_shortcut_are_combined() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
let engine = SearchEngine::open(directory.path()).unwrap();
|
||||
index_records(&engine, &[record()]);
|
||||
|
||||
let matching = engine
|
||||
.search_with(SearchOptions {
|
||||
query: "Ubuntu *.ISO".into(),
|
||||
limit: 10,
|
||||
..SearchOptions::default()
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(matching.total, 1);
|
||||
|
||||
let wrong_extension = engine
|
||||
.search_with(SearchOptions {
|
||||
query: "Ubuntu *.mkv".into(),
|
||||
limit: 10,
|
||||
..SearchOptions::default()
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(wrong_extension.total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arbitrary_wildcards_are_rejected_before_search() {
|
||||
let directory = TempDir::new().unwrap();
|
||||
|
||||
@@ -26,24 +26,41 @@ pub enum SearchError {
|
||||
Unavailable,
|
||||
#[error("仅支持 *.扩展名 形式的通配符搜索 例如 *.iso")]
|
||||
UnsupportedWildcard,
|
||||
#[error("一次搜索最多只能指定一个 *.扩展名 条件")]
|
||||
MultipleExtensions,
|
||||
}
|
||||
|
||||
pub(crate) fn extension_shortcut(query: &str) -> Result<Option<String>, SearchError> {
|
||||
let query = query.trim();
|
||||
if !query.contains('*') && !query.contains('?') {
|
||||
return Ok(None);
|
||||
pub(crate) struct ParsedSearchQuery {
|
||||
pub(crate) text: String,
|
||||
pub(crate) extension: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) fn parse_search_query(query: &str) -> Result<ParsedSearchQuery, SearchError> {
|
||||
let mut text = Vec::new();
|
||||
let mut extension = None;
|
||||
for token in query.split_whitespace() {
|
||||
if !token.contains('*') && !token.contains('?') {
|
||||
text.push(token);
|
||||
continue;
|
||||
}
|
||||
let Some(value) = token.strip_prefix("*.") else {
|
||||
return Err(SearchError::UnsupportedWildcard);
|
||||
};
|
||||
if value.is_empty()
|
||||
|| !value.chars().all(|character| {
|
||||
character.is_alphanumeric() || matches!(character, '_' | '-' | '+')
|
||||
})
|
||||
{
|
||||
return Err(SearchError::UnsupportedWildcard);
|
||||
}
|
||||
if extension.replace(value.to_lowercase()).is_some() {
|
||||
return Err(SearchError::MultipleExtensions);
|
||||
}
|
||||
}
|
||||
let Some(extension) = query.strip_prefix("*.") else {
|
||||
return Err(SearchError::UnsupportedWildcard);
|
||||
};
|
||||
if extension.is_empty()
|
||||
|| !extension
|
||||
.chars()
|
||||
.all(|character| character.is_alphanumeric() || matches!(character, '_' | '-' | '+'))
|
||||
{
|
||||
return Err(SearchError::UnsupportedWildcard);
|
||||
}
|
||||
Ok(Some(extension.to_lowercase()))
|
||||
Ok(ParsedSearchQuery {
|
||||
text: text.join(" "),
|
||||
extension,
|
||||
})
|
||||
}
|
||||
|
||||
impl SearchError {
|
||||
@@ -80,16 +97,30 @@ impl IndexingError {
|
||||
mod tests {
|
||||
use std::{io, sync::Arc};
|
||||
|
||||
use super::{IndexingError, SearchError, extension_shortcut};
|
||||
use super::{IndexingError, SearchError, parse_search_query};
|
||||
|
||||
#[test]
|
||||
fn extension_shortcut_accepts_only_one_extension_segment() {
|
||||
assert_eq!(extension_shortcut("*.ISO").unwrap().as_deref(), Some("iso"));
|
||||
assert_eq!(extension_shortcut("ubuntu").unwrap(), None);
|
||||
assert!(extension_shortcut("*iso").is_err());
|
||||
assert!(extension_shortcut("ubuntu*").is_err());
|
||||
assert!(extension_shortcut("*.tar.gz").is_err());
|
||||
assert!(extension_shortcut("file?.iso").is_err());
|
||||
fn query_parser_extracts_one_extension_from_plain_text() {
|
||||
let parsed = parse_search_query("Ubuntu 24.04 *.ISO").unwrap();
|
||||
assert_eq!(parsed.text, "Ubuntu 24.04");
|
||||
assert_eq!(parsed.extension.as_deref(), Some("iso"));
|
||||
|
||||
let reversed = parse_search_query("*.mkv 流浪地球").unwrap();
|
||||
assert_eq!(reversed.text, "流浪地球");
|
||||
assert_eq!(reversed.extension.as_deref(), Some("mkv"));
|
||||
|
||||
let plain = parse_search_query("ubuntu iso").unwrap();
|
||||
assert_eq!(plain.text, "ubuntu iso");
|
||||
assert_eq!(plain.extension, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_parser_rejects_arbitrary_or_multiple_wildcards() {
|
||||
assert!(parse_search_query("*iso").is_err());
|
||||
assert!(parse_search_query("ubuntu*").is_err());
|
||||
assert!(parse_search_query("*.tar.gz").is_err());
|
||||
assert!(parse_search_query("file?.iso").is_err());
|
||||
assert!(parse_search_query("*.iso *.img").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user