From c372aa9dd4082a51d152cc3a6ca2aa91614784c0 Mon Sep 17 00:00:00 2001 From: chuan Date: Tue, 11 Aug 2026 13:00:20 +0800 Subject: [PATCH] perf(search): use indexed extension shortcuts --- README.md | 2 +- TODOS.md | 2 +- src/search/README.md | 4 +- src/search/src/api/handlers.rs | 2 +- src/search/src/api/mod.rs | 12 +++ src/search/src/bin/dht-benchmark/workload.rs | 6 +- src/search/src/search/file_order.rs | 62 ++++---------- src/search/src/search/filter.rs | 88 +++----------------- src/search/src/search/indexer.rs | 36 +++++--- src/search/src/search/mod.rs | 32 ++++++- src/web/README.md | 2 +- src/web/src/pages/SearchPage.vue | 2 +- 12 files changed, 106 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index ae37bf0..cd8c74a 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ bun run build 大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大 -搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 通配符使用完整规范化文本字段 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer +搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 `*.iso` 一类扩展名快捷查询直接使用扩展名词项索引 搜索侧不执行正则或任意通配符扫描 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer 每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB diff --git a/TODOS.md b/TODOS.md index 932d7a8..15c1121 100644 --- a/TODOS.md +++ b/TODOS.md @@ -30,7 +30,7 @@ ### 验收标准 - [ ] 使用远端真实数据或等比例数据对比优化前后的索引体积 -- [ ] 普通文本通配符扩展名精确哈希和详情文件匹配测试保持通过 +- [ ] 普通文本扩展名快捷查询精确哈希和详情文件匹配测试保持通过 - [ ] 影子索引重建期间旧索引继续提供搜索且完成后能够原子切换 - [ ] 重建峰值空间不会触发磁盘只读保护 diff --git a/src/search/README.md b/src/search/README.md index 95b22e7..d8489b7 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -263,9 +263,9 @@ GET /torrents/{infohash}?file_offset=0&file_limit=100 搜索支持中文英文数字和文件名片段匹配 -默认搜索会自动识别不区分大小写的通配符并匹配名称 别名和文件路径 `*` 表示任意长度字符 `?` 表示一个字符 例如 `*.iso` 匹配所有以 `.iso` 结尾的已索引名称或文件路径 不含通配符时保持普通关键词和片段搜索 +搜索支持不区分大小写的单扩展名快捷语法 例如 `*.iso` 直接匹配包含 ISO 文件的已索引内容 普通关键词继续匹配标题别名文件名和路径片段 -搜索接口不接受正则模式 `mode` 参数 通配符最长 256 字节 +搜索接口不接受正则模式 `mode` 参数 也不接受 `*iso` `ubuntu*` `?` 或多段扩展名等任意通配符表达式 过滤参数还包括 `min_files` `max_files` `first_seen_after` `first_seen_before` `last_seen_after` `last_seen_before` `availability` 和 `heat` diff --git a/src/search/src/api/handlers.rs b/src/search/src/api/handlers.rs index 4e62774..1df0ce8 100644 --- a/src/search/src/api/handlers.rs +++ b/src/search/src/api/handlers.rs @@ -275,7 +275,7 @@ pub(crate) async fn search( return Err(ApiError::bad_request("搜索不再支持 mode 参数和正则表达式")); } if (request.q.contains('*') || request.q.contains('?')) && request.q.len() > 256 { - return Err(ApiError::bad_request("通配符不能超过 256 字节")); + return Err(ApiError::bad_request("扩展名快捷查询不能超过 256 字节")); } validate_range(request.min_size, request.max_size, "min_size", "max_size")?; validate_range( diff --git a/src/search/src/api/mod.rs b/src/search/src/api/mod.rs index 913cb90..3b4ccea 100644 --- a/src/search/src/api/mod.rs +++ b/src/search/src/api/mod.rs @@ -411,6 +411,18 @@ mod tests { .unwrap(); assert_eq!(json["total"], 1); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/search?q=%2Amkv") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = app .clone() .oneshot( diff --git a/src/search/src/bin/dht-benchmark/workload.rs b/src/search/src/bin/dht-benchmark/workload.rs index d9c5476..0b62230 100644 --- a/src/search/src/bin/dht-benchmark/workload.rs +++ b/src/search/src/bin/dht-benchmark/workload.rs @@ -1,4 +1,4 @@ -// 负责定义覆盖关键词路径哈希通配符排序和过滤的固定查询工作负载 +// 负责定义覆盖关键词路径哈希扩展名排序和过滤的固定查询工作负载 use std::error::Error; @@ -53,9 +53,9 @@ pub(crate) fn query_cases( }, }, QueryCase { - name: "文件通配符", + name: "扩展名快捷搜索", options: SearchOptions { - query: "ubuntu*desktop".into(), + query: "*.iso".into(), limit: 20, ..SearchOptions::default() }, diff --git a/src/search/src/search/file_order.rs b/src/search/src/search/file_order.rs index a4ca85b..66d0d7f 100644 --- a/src/search/src/search/file_order.rs +++ b/src/search/src/search/file_order.rs @@ -2,24 +2,17 @@ use std::cmp::Ordering; -use regex::{Regex, RegexBuilder}; use unicode_normalization::UnicodeNormalization; use crate::domain::TorrentFile; -use super::SearchSort; - -#[derive(Debug, thiserror::Error)] -pub(crate) enum FileOrderError { - #[error("文件相关性通配符无效: {0}")] - InvalidWildcard(#[from] regex::Error), -} +use super::{SearchError, SearchSort, extension_shortcut}; pub(crate) fn order_files( files: Vec, query: &str, sort: Option, -) -> Result, FileOrderError> { +) -> Result, SearchError> { let matcher = FileMatcher::compile(query)?; let secondary = SecondaryOrder::from_search_sort(sort); let mut ranked: Vec<_> = files @@ -68,21 +61,17 @@ impl RankedFile { enum FileMatcher { None, Plain(Vec), - Pattern(Regex), + Extension(String), } impl FileMatcher { - fn compile(query: &str) -> Result { + fn compile(query: &str) -> Result { let query = query.trim(); - if query.is_empty() || query == "*" { + if query.is_empty() { return Ok(Self::None); } - if query.contains('*') || query.contains('?') { - return RegexBuilder::new(&wildcard_regex(&normalize(query))) - .unicode(true) - .build() - .map(Self::Pattern) - .map_err(FileOrderError::from); + if let Some(extension) = extension_shortcut(query)? { + return Ok(Self::Extension(format!(".{extension}"))); } let terms = normalize(query) .split_whitespace() @@ -95,7 +84,7 @@ impl FileMatcher { match self { Self::None => false, Self::Plain(terms) => terms.iter().all(|term| value.contains(term)), - Self::Pattern(regex) => regex.is_match(value), + Self::Extension(extension) => value.ends_with(extension), } } } @@ -129,29 +118,6 @@ fn normalize(value: &str) -> String { value.nfkc().collect::().to_lowercase() } -fn wildcard_regex(pattern: &str) -> String { - let mut output = String::from("^"); - let mut escaped = false; - for character in pattern.chars() { - if escaped { - output.push_str(®ex::escape(&character.to_string())); - escaped = false; - } else { - match character { - '\\' => escaped = true, - '*' => output.push_str(".*"), - '?' => output.push('.'), - _ => output.push_str(®ex::escape(&character.to_string())), - } - } - } - if escaped { - output.push_str(r"\\"); - } - output.push('$'); - output -} - #[cfg(test)] mod tests { use super::*; @@ -192,9 +158,15 @@ mod tests { } #[test] - fn wildcard_promotes_matching_files() { + fn extension_shortcut_promotes_matching_files() { let files = vec![file("a/movie.mkv", 1), file("b/movie.iso", 2)]; - let wildcard = order_files(files, "*.iso", Some(SearchSort::SizeAsc)).unwrap(); - assert_eq!(wildcard[0].path, "b/movie.iso"); + let ordered = order_files(files, "*.ISO", Some(SearchSort::SizeAsc)).unwrap(); + assert_eq!(ordered[0].path, "b/movie.iso"); + } + + #[test] + fn unsupported_wildcard_is_rejected() { + let files = vec![file("movie.iso", 1)]; + assert!(order_files(files, "*iso", None).is_err()); } } diff --git a/src/search/src/search/filter.rs b/src/search/src/search/filter.rs index 971f794..8c1d10c 100644 --- a/src/search/src/search/filter.rs +++ b/src/search/src/search/filter.rs @@ -4,7 +4,7 @@ use std::ops::Bound; use tantivy::{ Term, - query::{AllQuery, BooleanQuery, BoostQuery, Occur, Query, RangeQuery, RegexQuery, TermQuery}, + query::{AllQuery, BooleanQuery, BoostQuery, Occur, Query, RangeQuery, TermQuery}, schema::IndexRecordOption, }; use unicode_normalization::UnicodeNormalization; @@ -12,6 +12,7 @@ use unicode_normalization::UnicodeNormalization; use super::{ SearchError, document::availability_number, + extension_shortcut, query::{SearchOptions, SearchSort}, schema::SearchFields, }; @@ -27,14 +28,15 @@ pub(crate) fn prepare( ) -> Result { let mut clauses: Vec> = Vec::new(); let query_text = options.query.trim(); - if query_text.is_empty() || query_text == "*" { + if query_text.is_empty() { clauses.push(Box::new(AllQuery)); + } else if let Some(extension) = extension_shortcut(query_text)? { + clauses.push(Box::new(TermQuery::new( + Term::from_field_text(fields.extensions, &extension), + IndexRecordOption::Basic, + ))); } else { - clauses.push(if has_wildcard_syntax(query_text) { - wildcard_query(query_text, fields)? - } else { - text_query(query_text, fields)? - }); + clauses.push(text_query(query_text, fields)?); } if let Some(content_key) = options.content_key { clauses.push(Box::new(TermQuery::new( @@ -98,8 +100,8 @@ pub(crate) fn prepare( } else { Box::new(BooleanQuery::intersection(clauses)) }; - let sort = options.sort.unwrap_or_else(|| { - if query_text.is_empty() || query_text == "*" { + let sort = options.sort.unwrap_or({ + if query_text.is_empty() { SearchSort::Latest } else { SearchSort::Relevance @@ -108,50 +110,6 @@ pub(crate) fn prepare( Ok(PreparedQuery { query, sort }) } -fn wildcard_query(pattern: &str, fields: SearchFields) -> Result, SearchError> { - Ok(Box::new(RegexQuery::from_pattern( - &wildcard_pattern(pattern), - fields.regex_text, - )?)) -} - -fn has_wildcard_syntax(pattern: &str) -> bool { - pattern.contains('*') || pattern.contains('?') -} - -fn wildcard_pattern(pattern: &str) -> String { - let pattern = normalize_text(pattern); - let mut regex = String::with_capacity(pattern.len()); - let mut escaped = false; - for character in pattern.chars() { - if escaped { - push_regex_literal(&mut regex, character); - escaped = false; - continue; - } - match character { - '\\' => escaped = true, - '*' => regex.push_str(".*"), - '?' => regex.push('.'), - literal => push_regex_literal(&mut regex, literal), - } - } - if escaped { - push_regex_literal(&mut regex, '\\'); - } - regex -} - -fn push_regex_literal(regex: &mut String, character: char) { - if matches!( - character, - '\\' | '.' | '+' | '*' | '?' | '(' | ')' | '|' | '[' | ']' | '{' | '}' | '^' | '$' - ) { - regex.push('\\'); - } - regex.push(character); -} - fn text_query(query: &str, fields: SearchFields) -> Result, SearchError> { let normalized = normalize_text(query.trim()); if normalized.len() == 40 && normalized.bytes().all(|byte| byte.is_ascii_hexdigit()) { @@ -212,15 +170,6 @@ fn text_query(query: &str, fields: SearchFields) -> Result, Searc if let Some(query) = path_query(fields.files_text, &part) { alternatives.push((Occur::Should, query)); } - if alternatives.is_empty() { - alternatives.push(( - Occur::Should, - Box::new(RegexQuery::from_pattern( - &format!(".*{}.*", regex::escape(&part)), - fields.regex_text, - )?), - )); - } required.push(( Occur::Must, Box::new(BooleanQuery::new(alternatives)) as Box, @@ -308,18 +257,3 @@ fn add_range( .unwrap_or(Bound::Unbounded); clauses.push(Box::new(RangeQuery::new(lower, upper))); } - -#[cfg(test)] -mod tests { - use super::{has_wildcard_syntax, wildcard_pattern}; - - #[test] - fn wildcard_conversion_is_case_insensitive_and_escapes_regex_syntax() { - assert_eq!(wildcard_pattern("*.ISO"), r".*\.iso"); - assert_eq!(wildcard_pattern("file?.[ch]"), r"file.\.\[ch\]"); - assert_eq!(wildcard_pattern(r"literal\*name"), r"literal\*name"); - assert!(has_wildcard_syntax("*.iso")); - assert!(has_wildcard_syntax("file?.mkv")); - assert!(has_wildcard_syntax(r"literal\*name")); - } -} diff --git a/src/search/src/search/indexer.rs b/src/search/src/search/indexer.rs index 2ef26f8..fdf2768 100644 --- a/src/search/src/search/indexer.rs +++ b/src/search/src/search/indexer.rs @@ -674,21 +674,19 @@ mod tests { } #[test] - fn wildcard_matches_names_and_file_extensions() { + fn extension_shortcut_uses_indexed_file_extensions() { let directory = TempDir::new().unwrap(); let engine = SearchEngine::open(directory.path()).unwrap(); index_records(&engine, &[record()]); - for pattern in ["*.iso", "Ubuntu*", "ubuntu.?so"] { - let page = engine - .search_with(SearchOptions { - query: pattern.into(), - limit: 10, - ..SearchOptions::default() - }) - .unwrap(); - assert_eq!(page.total, 1, "pattern {pattern}"); - } + let page = engine + .search_with(SearchOptions { + query: "*.ISO".into(), + limit: 10, + ..SearchOptions::default() + }) + .unwrap(); + assert_eq!(page.total, 1); let missing = engine .search_with(SearchOptions { query: "*.img".into(), @@ -699,6 +697,22 @@ mod tests { assert_eq!(missing.total, 0); } + #[test] + fn arbitrary_wildcards_are_rejected_before_search() { + let directory = TempDir::new().unwrap(); + let engine = SearchEngine::open(directory.path()).unwrap(); + index_records(&engine, &[record()]); + + for pattern in ["*iso", "Ubuntu*", "ubuntu.?so", "*"] { + let result = engine.search_with(SearchOptions { + query: pattern.into(), + limit: 10, + ..SearchOptions::default() + }); + assert!(result.is_err(), "pattern {pattern}"); + } + } + #[test] fn equal_content_is_collapsed_and_aliases_remain_searchable() { let directory = TempDir::new().unwrap(); diff --git a/src/search/src/search/mod.rs b/src/search/src/search/mod.rs index 075b008..b251bef 100644 --- a/src/search/src/search/mod.rs +++ b/src/search/src/search/mod.rs @@ -24,6 +24,26 @@ pub enum SearchError { MissingField(&'static str), #[error("搜索索引尚未可用")] Unavailable, + #[error("仅支持 *.扩展名 形式的通配符搜索 例如 *.iso")] + UnsupportedWildcard, +} + +pub(crate) fn extension_shortcut(query: &str) -> Result, SearchError> { + let query = query.trim(); + if !query.contains('*') && !query.contains('?') { + return Ok(None); + } + let Some(extension) = query.strip_prefix("*.") else { + return Err(SearchError::UnsupportedWildcard); + }; + if extension.is_empty() + || !extension + .chars() + .all(|character| character.is_alphanumeric() || matches!(character, '_' | '-' | '+')) + { + return Err(SearchError::UnsupportedWildcard); + } + Ok(Some(extension.to_lowercase())) } impl SearchError { @@ -60,7 +80,17 @@ impl IndexingError { mod tests { use std::{io, sync::Arc}; - use super::{IndexingError, SearchError}; + use super::{IndexingError, SearchError, extension_shortcut}; + + #[test] + fn extension_shortcut_accepts_only_one_extension_segment() { + assert_eq!(extension_shortcut("*.ISO").unwrap().as_deref(), Some("iso")); + assert_eq!(extension_shortcut("ubuntu").unwrap(), None); + assert!(extension_shortcut("*iso").is_err()); + assert!(extension_shortcut("ubuntu*").is_err()); + assert!(extension_shortcut("*.tar.gz").is_err()); + assert!(extension_shortcut("file?.iso").is_err()); + } #[test] fn permission_denied_index_error_is_retryable() { diff --git a/src/web/README.md b/src/web/README.md index 4496250..60ae713 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -50,7 +50,7 @@ Axum 根据配置中的 `http.web_dir` 提供静态资源和单页回退 不需 ## 已实现功能 - 关键词 文件名片段和精确 infohash 搜索 -- 支持普通关键词 通配符和精确 infohash 搜索 +- 支持普通关键词 `*.iso` 单扩展名快捷语法和精确 infohash 搜索 - 相关度 时间 热度 大小和发现次数排序 - 有上限的结果分页和 URL 查询恢复 - 基于 reka-ui 的搜索结果数字分页和浏览器持久化每页数量选择 diff --git a/src/web/src/pages/SearchPage.vue b/src/web/src/pages/SearchPage.vue index fbc1772..049d970 100644 --- a/src/web/src/pages/SearchPage.vue +++ b/src/web/src/pages/SearchPage.vue @@ -237,7 +237,7 @@ onBeforeUnmount(() => {
正在搜索

无法完成搜索

{{ error }}

-

没有找到匹配内容

尝试其他关键词或通配符

+

没有找到匹配内容

尝试其他关键词或 *.iso 扩展名查询