From 8d1406a20ccb5a9315891df1920fdc09ef462f8b Mon Sep 17 00:00:00 2001 From: chuan Date: Tue, 11 Aug 2026 13:21:38 +0800 Subject: [PATCH] feat(search): combine text and extension queries --- README.md | 2 +- TODOS.md | 2 +- src/search/README.md | 2 +- src/search/src/api/mod.rs | 16 ++++++ src/search/src/search/file_order.rs | 73 +++++++++++++++++---------- src/search/src/search/filter.rs | 13 +++-- src/search/src/search/indexer.rs | 25 ++++++++++ src/search/src/search/mod.rs | 77 ++++++++++++++++++++--------- 8 files changed, 153 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index cd8c74a..bda5d3e 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ bun run build 大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大 -搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer +搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引并可与普通关键词组合 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer 每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB diff --git a/TODOS.md b/TODOS.md index 15c1121..4c14e35 100644 --- a/TODOS.md +++ b/TODOS.md @@ -30,7 +30,7 @@ ### 验收标准 - [ ] 使用远端真实数据或等比例数据对比优化前后的索引体积 -- [ ] 普通文本扩展名快捷查询精确哈希和详情文件匹配测试保持通过 +- [ ] 普通文本与扩展名组合查询精确哈希和详情文件匹配测试保持通过 - [ ] 影子索引重建期间旧索引继续提供搜索且完成后能够原子切换 - [ ] 重建峰值空间不会触发磁盘只读保护 diff --git a/src/search/README.md b/src/search/README.md index d8489b7..5ff6eed 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -263,7 +263,7 @@ GET /torrents/{infohash}?file_offset=0&file_limit=100 搜索支持中文英文数字和文件名片段匹配 -搜索支持不区分大小写的单扩展名快捷语法 例如 `*.iso` 直接匹配包含 ISO 文件的已索引内容 普通关键词继续匹配标题别名文件名和路径片段 +搜索支持不区分大小写的单扩展名快捷语法 例如 `*.iso` 直接匹配包含 ISO 文件的已索引内容 `Ubuntu *.iso` 表示普通关键词和扩展名条件同时满足 普通关键词继续匹配标题别名文件名和路径片段 搜索接口不接受正则模式 `mode` 参数 也不接受 `*iso` `ubuntu*` `?` 或多段扩展名等任意通配符表达式 diff --git a/src/search/src/api/mod.rs b/src/search/src/api/mod.rs index 3b4ccea..84885a3 100644 --- a/src/search/src/api/mod.rs +++ b/src/search/src/api/mod.rs @@ -411,6 +411,22 @@ mod tests { .unwrap(); assert_eq!(json["total"], 1); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/search?q=Example%20%2A.mkv") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let json: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(json["total"], 1); + let response = app .clone() .oneshot( diff --git a/src/search/src/search/file_order.rs b/src/search/src/search/file_order.rs index 66d0d7f..33ba421 100644 --- a/src/search/src/search/file_order.rs +++ b/src/search/src/search/file_order.rs @@ -6,7 +6,7 @@ use unicode_normalization::UnicodeNormalization; use crate::domain::TorrentFile; -use super::{SearchError, SearchSort, extension_shortcut}; +use super::{SearchError, SearchSort, parse_search_query}; pub(crate) fn order_files( files: Vec, @@ -42,13 +42,7 @@ impl RankedFile { fn new(file: TorrentFile, matcher: &FileMatcher) -> Self { let path = normalize(&file.path).replace('\\', "/"); let name = path.rsplit('/').next().unwrap_or(path.as_str()).to_owned(); - let relevance = if matcher.is_match(&name) { - 2 - } else if matcher.is_match(&path) { - 1 - } else { - 0 - }; + let relevance = matcher.relevance(&name, &path); Self { file, name, @@ -58,33 +52,45 @@ impl RankedFile { } } -enum FileMatcher { - None, - Plain(Vec), - Extension(String), +struct FileMatcher { + terms: Vec, + extension: Option, } impl FileMatcher { fn compile(query: &str) -> Result { - let query = query.trim(); - if query.is_empty() { - return Ok(Self::None); - } - if let Some(extension) = extension_shortcut(query)? { - return Ok(Self::Extension(format!(".{extension}"))); - } - let terms = normalize(query) + let parsed = parse_search_query(query)?; + let terms = normalize(&parsed.text) .split_whitespace() .map(ToOwned::to_owned) .collect(); - Ok(Self::Plain(terms)) + Ok(Self { + terms, + extension: parsed.extension.map(|value| format!(".{value}")), + }) } - fn is_match(&self, value: &str) -> bool { - match self { - Self::None => false, - Self::Plain(terms) => terms.iter().all(|term| value.contains(term)), - Self::Extension(extension) => value.ends_with(extension), + fn relevance(&self, name: &str, path: &str) -> u8 { + let text_relevance = if self.terms.is_empty() { + 0 + } else if self.terms.iter().all(|term| name.contains(term)) { + 2 + } else if self.terms.iter().all(|term| path.contains(term)) { + 1 + } else { + 0 + }; + let extension_matches = self + .extension + .as_ref() + .is_some_and(|extension| name.ends_with(extension)); + match (text_relevance, extension_matches) { + (2, true) => 5, + (1, true) => 4, + (2, false) => 3, + (1, false) => 2, + (0, true) => 1, + _ => 0, } } } @@ -169,4 +175,19 @@ mod tests { let files = vec![file("movie.iso", 1)]; assert!(order_files(files, "*iso", None).is_err()); } + + #[test] + fn combined_query_prefers_files_matching_text_and_extension() { + let files = vec![ + file("ubuntu.txt", 1), + file("disk.iso", 2), + file("ubuntu.iso", 3), + file("other.bin", 4), + ]; + let ordered = order_files(files, "Ubuntu *.ISO", None).unwrap(); + assert_eq!(ordered[0].path, "ubuntu.iso"); + assert_eq!(ordered[1].path, "ubuntu.txt"); + assert_eq!(ordered[2].path, "disk.iso"); + assert_eq!(ordered[3].path, "other.bin"); + } } diff --git a/src/search/src/search/filter.rs b/src/search/src/search/filter.rs index 8c1d10c..58e509a 100644 --- a/src/search/src/search/filter.rs +++ b/src/search/src/search/filter.rs @@ -12,7 +12,7 @@ use unicode_normalization::UnicodeNormalization; use super::{ SearchError, document::availability_number, - extension_shortcut, + parse_search_query, query::{SearchOptions, SearchSort}, schema::SearchFields, }; @@ -28,15 +28,18 @@ pub(crate) fn prepare( ) -> Result { let mut clauses: Vec> = Vec::new(); let query_text = options.query.trim(); - if query_text.is_empty() { + let parsed = parse_search_query(query_text)?; + if parsed.text.is_empty() && parsed.extension.is_none() { clauses.push(Box::new(AllQuery)); - } else if let Some(extension) = extension_shortcut(query_text)? { + } + if !parsed.text.is_empty() { + clauses.push(text_query(&parsed.text, fields)?); + } + if let Some(extension) = parsed.extension { clauses.push(Box::new(TermQuery::new( Term::from_field_text(fields.extensions, &extension), IndexRecordOption::Basic, ))); - } else { - clauses.push(text_query(query_text, fields)?); } if let Some(content_key) = options.content_key { clauses.push(Box::new(TermQuery::new( diff --git a/src/search/src/search/indexer.rs b/src/search/src/search/indexer.rs index fdf2768..45002df 100644 --- a/src/search/src/search/indexer.rs +++ b/src/search/src/search/indexer.rs @@ -697,6 +697,31 @@ mod tests { assert_eq!(missing.total, 0); } + #[test] + fn text_and_extension_shortcut_are_combined() { + let directory = TempDir::new().unwrap(); + let engine = SearchEngine::open(directory.path()).unwrap(); + index_records(&engine, &[record()]); + + let matching = engine + .search_with(SearchOptions { + query: "Ubuntu *.ISO".into(), + limit: 10, + ..SearchOptions::default() + }) + .unwrap(); + assert_eq!(matching.total, 1); + + let wrong_extension = engine + .search_with(SearchOptions { + query: "Ubuntu *.mkv".into(), + limit: 10, + ..SearchOptions::default() + }) + .unwrap(); + assert_eq!(wrong_extension.total, 0); + } + #[test] fn arbitrary_wildcards_are_rejected_before_search() { let directory = TempDir::new().unwrap(); diff --git a/src/search/src/search/mod.rs b/src/search/src/search/mod.rs index b251bef..b32f058 100644 --- a/src/search/src/search/mod.rs +++ b/src/search/src/search/mod.rs @@ -26,24 +26,41 @@ pub enum SearchError { Unavailable, #[error("仅支持 *.扩展名 形式的通配符搜索 例如 *.iso")] UnsupportedWildcard, + #[error("一次搜索最多只能指定一个 *.扩展名 条件")] + MultipleExtensions, } -pub(crate) fn extension_shortcut(query: &str) -> Result, SearchError> { - let query = query.trim(); - if !query.contains('*') && !query.contains('?') { - return Ok(None); +pub(crate) struct ParsedSearchQuery { + pub(crate) text: String, + pub(crate) extension: Option, +} + +pub(crate) fn parse_search_query(query: &str) -> Result { + let mut text = Vec::new(); + let mut extension = None; + for token in query.split_whitespace() { + if !token.contains('*') && !token.contains('?') { + text.push(token); + continue; + } + let Some(value) = token.strip_prefix("*.") else { + return Err(SearchError::UnsupportedWildcard); + }; + if value.is_empty() + || !value.chars().all(|character| { + character.is_alphanumeric() || matches!(character, '_' | '-' | '+') + }) + { + return Err(SearchError::UnsupportedWildcard); + } + if extension.replace(value.to_lowercase()).is_some() { + return Err(SearchError::MultipleExtensions); + } } - let Some(extension) = query.strip_prefix("*.") else { - return Err(SearchError::UnsupportedWildcard); - }; - if extension.is_empty() - || !extension - .chars() - .all(|character| character.is_alphanumeric() || matches!(character, '_' | '-' | '+')) - { - return Err(SearchError::UnsupportedWildcard); - } - Ok(Some(extension.to_lowercase())) + Ok(ParsedSearchQuery { + text: text.join(" "), + extension, + }) } impl SearchError { @@ -80,16 +97,30 @@ impl IndexingError { mod tests { use std::{io, sync::Arc}; - use super::{IndexingError, SearchError, extension_shortcut}; + use super::{IndexingError, SearchError, parse_search_query}; #[test] - fn extension_shortcut_accepts_only_one_extension_segment() { - assert_eq!(extension_shortcut("*.ISO").unwrap().as_deref(), Some("iso")); - assert_eq!(extension_shortcut("ubuntu").unwrap(), None); - assert!(extension_shortcut("*iso").is_err()); - assert!(extension_shortcut("ubuntu*").is_err()); - assert!(extension_shortcut("*.tar.gz").is_err()); - assert!(extension_shortcut("file?.iso").is_err()); + fn query_parser_extracts_one_extension_from_plain_text() { + let parsed = parse_search_query("Ubuntu 24.04 *.ISO").unwrap(); + assert_eq!(parsed.text, "Ubuntu 24.04"); + assert_eq!(parsed.extension.as_deref(), Some("iso")); + + let reversed = parse_search_query("*.mkv 流浪地球").unwrap(); + assert_eq!(reversed.text, "流浪地球"); + assert_eq!(reversed.extension.as_deref(), Some("mkv")); + + let plain = parse_search_query("ubuntu iso").unwrap(); + assert_eq!(plain.text, "ubuntu iso"); + assert_eq!(plain.extension, None); + } + + #[test] + fn query_parser_rejects_arbitrary_or_multiple_wildcards() { + assert!(parse_search_query("*iso").is_err()); + assert!(parse_search_query("ubuntu*").is_err()); + assert!(parse_search_query("*.tar.gz").is_err()); + assert!(parse_search_query("file?.iso").is_err()); + assert!(parse_search_query("*.iso *.img").is_err()); } #[test]