tools/dht
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(search): remove regex query mode

Browse Source
This commit is contained in:
chuan
2026-08-11 12:50:08 +08:00
parent 753534f53c
commit 42239f91b3
16 changed files with 43 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -3
View File
@@ -141,15 +141,14 @@ bun run build
| 英文关键词 | 7.746 ms |
| 文件路径片段 | 1.063 ms |
| 精确 infohash | 0.031 ms |
| 有限状态正则 | 675.803 ms |
| 最近收录排序 | 3.367 ms |
| 大小扩展名过滤 | 6.641 ms |
百万级基准中普通搜索过滤排序精确哈希索引吞吐磁盘和峰值内存均达到当前目标 大命中集合正则仍是继续扩大规模前最值得优化的查询路径
百万级基准中普通搜索过滤排序精确哈希索引吞吐磁盘和峰值内存均达到当前目标
大型种子会先按文件大小降序和规范化路径稳定排序 最多索引 2048 个文件且完整路径文本总量不超过 256 KiB 以优先覆盖主体内容并限制极端 Metadata 的索引放大
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 正则和通配符继续使用完整规范化文本字段 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
搜索索引对标题使用最多 10 字符的有限 N-Gram 对文件名使用最多 8 字符的有限 N-Gram 完整路径仅按目录段和单词分词 普通文本查询会按各字段策略拆分 通配符使用完整规范化文本字段 索引不保存未使用的词位置信息并在首次写入前延迟创建 Tantivy writer
每条记录包含 2048 个文件的极端基准中 450 个内容文档的 Tantivy 索引为 19.84 MiB 平均每文档 46.2 KiB 峰值内存为 113.08 MiB
TODOS.md
+1 -1
View File
@@ -30,7 +30,7 @@
### 验收标准
- [ ] 使用远端真实数据或等比例数据对比优化前后的索引体积
- [ ] 普通文本通配符正则扩展名精确哈希和详情文件匹配测试保持通过
- [ ] 普通文本通配符扩展名精确哈希和详情文件匹配测试保持通过
- [ ] 影子索引重建期间旧索引继续提供搜索且完成后能够原子切换
- [ ] 重建峰值空间不会触发磁盘只读保护
src/search/README.md
+1 -2
View File
@@ -255,7 +255,6 @@ GET /search?q=ubuntu&offset=0&limit=20
GET /search?q=%2A.iso
GET /search?q=&min_size=1048576&max_size=10737418240&extension=mkv
GET /search?q=流浪地球&min_files=1&availability=active&heat=hot&sort=heat
GET /search?q=%5ES%5Cd%7B2%7DE%5Cd%7B2%7D&mode=regex
GET /contents/{content_key}?offset=0&limit=20
GET /torrents/{infohash}?file_offset=0&file_limit=100
```
@@ -266,7 +265,7 @@ GET /torrents/{infohash}?file_offset=0&file_limit=100
默认搜索会自动识别不区分大小写的通配符并匹配名称 别名和文件路径 `*` 表示任意长度字符 `?` 表示一个字符 例如 `*.iso` 匹配所有以 `.iso` 结尾的已索引名称或文件路径 不含通配符时保持普通关键词和片段搜索
设置 `mode=regex` 后查询文本作为不区分大小写的正则表达式匹配名称 别名和文件路径 通配符和正则最长 256 字节并由 Tantivy 有限状态自动机执行
搜索接口不接受正则模式 `mode` 参数 通配符最长 256 字节
过滤参数还包括 `min_files` `max_files` `first_seen_after` `first_seen_before` `last_seen_after` `last_seen_before` `availability` 和 `heat`
src/search/src/api/handlers.rs
+13 -12
View File
@@ -7,7 +7,7 @@ use std::{
use crate::{
domain::{InfoHash, MetadataRejectionReason},
search::{SearchMode, SearchOptions, SearchPage, SearchSort},
search::{SearchOptions, SearchPage, SearchSort},
storage::VerificationPriority,
};
use axum::{
@@ -271,10 +271,11 @@ pub(crate) async fn search(
if request.q.len() > 512 {
return Err(ApiError::bad_request("查询文本不能超过 512 字节"));
}
if (request.mode != SearchMode::Text || request.q.contains('*') || request.q.contains('?'))
&& request.q.len() > 256
{
return Err(ApiError::bad_request("通配符或正则表达式不能超过 256 字节"));
if request.mode.is_some() {
return Err(ApiError::bad_request("搜索不再支持 mode 参数和正则表达式"));
}
if (request.q.contains('*') || request.q.contains('?')) && request.q.len() > 256 {
return Err(ApiError::bad_request("通配符不能超过 256 字节"));
}
validate_range(request.min_size, request.max_size, "min_size", "max_size")?;
validate_range(
@@ -296,11 +297,7 @@ pub(crate) async fn search(
"last_seen_before",
)?;
let mut query = request.q;
let content_key = if request.mode == SearchMode::Text {
InfoHash::from_str(query.trim()).ok()
} else {
None
};
let content_key = InfoHash::from_str(query.trim()).ok();
let content_key = if let Some(info_hash) = content_key {
let repository = state.repository.clone();
let record = tokio::task::spawn_blocking(move || repository.get_visible(info_hash))
@@ -325,7 +322,6 @@ pub(crate) async fn search(
let mut page = tokio::task::spawn_blocking(move || {
search.search_with(SearchOptions {
query,
mode: request.mode,
offset: request.offset,
limit: request.limit,
min_size: request.min_size,
@@ -435,6 +431,11 @@ pub(crate) async fn torrent(
if request.file_offset > 1_000_000 {
return Err(ApiError::bad_request("file_offset 不能超过 1000000"));
}
if request.mode.is_some() {
return Err(ApiError::bad_request(
"详情接口不再支持 mode 参数和正则表达式",
));
}
let verification = state.crawler.verification();
let info_hash =
InfoHash::from_str(&info_hash).map_err(|error| ApiError::bad_request(error.to_string()))?;
@@ -444,7 +445,7 @@ pub(crate) async fn torrent(
.map_err(|error| ApiError::internal(error.to_string()))?
.ok_or_else(|| ApiError::not_found("没有找到该 infohash"))?;
let mut record = record;
record.files = crate::search::order_files(record.files, &request.q, request.mode, request.sort)
record.files = crate::search::order_files(record.files, &request.q, request.sort)
.map_err(|error| ApiError::bad_request(error.to_string()))?;
if let Some(verification) = &verification {
verification
src/search/src/api/mod.rs
+1 -17
View File
@@ -393,11 +393,7 @@ mod tests {
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let json: serde_json::Value =
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
.unwrap();
assert_eq!(json["total"], 1);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let response = app
.clone()
@@ -415,18 +411,6 @@ mod tests {
.unwrap();
assert_eq!(json["total"], 1);
let response = app
.clone()
.oneshot(
Request::builder()
.uri("/search?q=%5B&mode=regex")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let response = app
.clone()
.oneshot(
src/search/src/api/request.rs
+3 -5
View File
@@ -2,7 +2,7 @@
use crate::{
domain::{AvailabilityStatus, HeatLevel},
search::{SearchMode, SearchSort},
search::SearchSort,
};
use serde::Deserialize;
@@ -19,8 +19,7 @@ fn default_limit() -> usize {
pub(crate) struct SearchRequest {
#[serde(default)]
pub(crate) q: String,
#[serde(default)]
pub(crate) mode: SearchMode,
pub(crate) mode: Option<String>,
#[serde(default)]
pub(crate) offset: usize,
#[serde(default = "default_limit")]
@@ -55,8 +54,7 @@ pub(crate) struct TorrentRequest {
pub(crate) file_limit: usize,
#[serde(default)]
pub(crate) q: String,
#[serde(default)]
pub(crate) mode: SearchMode,
pub(crate) mode: Option<String>,
pub(crate) sort: Option<SearchSort>,
}
src/search/src/bin/dht-benchmark/workload.rs
+4 -5
View File
@@ -1,8 +1,8 @@
// 负责定义覆盖关键词路径哈希正则排序和过滤的固定查询工作负载
// 负责定义覆盖关键词路径哈希通配符排序和过滤的固定查询工作负载
use std::error::Error;
use dht_search::search::{SearchMode, SearchOptions, SearchSort};
use dht_search::search::{SearchOptions, SearchSort};
use super::dataset::generate_record;
@@ -53,10 +53,9 @@ pub(crate) fn query_cases(
},
},
QueryCase {
name: "有限状态正则",
name: "文件通配符",
options: SearchOptions {
query: "ubuntu.*desktop".into(),
mode: SearchMode::Regex,
query: "ubuntu*desktop".into(),
limit: 20,
..SearchOptions::default()
},
src/search/src/search/file_order.rs
+9 -28
View File
@@ -7,21 +7,20 @@ use unicode_normalization::UnicodeNormalization;
use crate::domain::TorrentFile;
use super::{SearchMode, SearchSort};
use super::SearchSort;
#[derive(Debug, thiserror::Error)]
pub(crate) enum FileOrderError {
#[error("文件相关性正则表达式无效: {0}")]
InvalidRegex(#[from] regex::Error),
#[error("文件相关性通配符无效: {0}")]
InvalidWildcard(#[from] regex::Error),
}
pub(crate) fn order_files(
files: Vec<TorrentFile>,
query: &str,
mode: SearchMode,
sort: Option<SearchSort>,
) -> Result<Vec<TorrentFile>, FileOrderError> {
let matcher = FileMatcher::compile(query, mode)?;
let matcher = FileMatcher::compile(query)?;
let secondary = SecondaryOrder::from_search_sort(sort);
let mut ranked: Vec<_> = files
.into_iter()
@@ -73,19 +72,11 @@ enum FileMatcher {
}
impl FileMatcher {
fn compile(query: &str, mode: SearchMode) -> Result<Self, FileOrderError> {
fn compile(query: &str) -> Result<Self, FileOrderError> {
let query = query.trim();
if query.is_empty() || query == "*" {
return Ok(Self::None);
}
if mode == SearchMode::Regex {
return RegexBuilder::new(&query.to_lowercase())
.case_insensitive(true)
.unicode(true)
.build()
.map(Self::Pattern)
.map_err(FileOrderError::from);
}
if query.contains('*') || query.contains('?') {
return RegexBuilder::new(&wildcard_regex(&normalize(query)))
.unicode(true)
@@ -179,7 +170,7 @@ mod tests {
file("ubuntu/docs/a.txt", 2),
file("a/ubuntu.iso", 3),
];
let ordered = order_files(files, "ubuntu", SearchMode::Text, None).unwrap();
let ordered = order_files(files, "ubuntu", None).unwrap();
assert_eq!(ordered[0].path, "a/ubuntu.iso");
assert_eq!(ordered[1].path, "ubuntu/docs/a.txt");
assert_eq!(ordered[2].path, "z/readme.txt");
@@ -193,8 +184,7 @@ mod tests {
file("match-large.iso", 10),
file("unrelated-small.bin", 2),
];
let ordered =
order_files(files, "match", SearchMode::Text, Some(SearchSort::SizeDesc)).unwrap();
let ordered = order_files(files, "match", Some(SearchSort::SizeDesc)).unwrap();
assert_eq!(ordered[0].path, "match-large.iso");
assert_eq!(ordered[1].path, "match-small.iso");
assert_eq!(ordered[2].path, "unrelated-large.bin");
@@ -202,18 +192,9 @@ mod tests {
}
#[test]
fn wildcard_and_regex_follow_search_mode() {
fn wildcard_promotes_matching_files() {
let files = vec![file("a/movie.mkv", 1), file("b/movie.iso", 2)];
let wildcard = order_files(
files.clone(),
"*.iso",
SearchMode::Text,
Some(SearchSort::SizeAsc),
)
.unwrap();
let wildcard = order_files(files, "*.iso", Some(SearchSort::SizeAsc)).unwrap();
assert_eq!(wildcard[0].path, "b/movie.iso");
let regex = order_files(files, r"movie\.mkv$", SearchMode::Regex, None).unwrap();
assert_eq!(regex[0].path, "a/movie.mkv");
}
}
src/search/src/search/filter.rs
+5 -32
View File
@@ -12,7 +12,7 @@ use unicode_normalization::UnicodeNormalization;
use super::{
SearchError,
document::availability_number,
query::{SearchMode, SearchOptions, SearchSort},
query::{SearchOptions, SearchSort},
schema::SearchFields,
};
@@ -30,12 +30,10 @@ pub(crate) fn prepare(
if query_text.is_empty() || query_text == "*" {
clauses.push(Box::new(AllQuery));
} else {
clauses.push(match options.mode {
SearchMode::Text if has_wildcard_syntax(query_text) => {
wildcard_query(query_text, fields)?
}
SearchMode::Text => text_query(query_text, fields)?,
SearchMode::Regex => regex_query(query_text, fields)?,
clauses.push(if has_wildcard_syntax(query_text) {
wildcard_query(query_text, fields)?
} else {
text_query(query_text, fields)?
});
}
if let Some(content_key) = options.content_key {
@@ -110,31 +108,6 @@ pub(crate) fn prepare(
Ok(PreparedQuery { query, sort })
}
fn regex_query(pattern: &str, fields: SearchFields) -> Result<Box<dyn Query>, SearchError> {
let mut pattern = pattern.to_lowercase();
let anchored_start = pattern.starts_with('^');
if anchored_start {
pattern.remove(0);
}
let anchored_end = pattern.ends_with('$')
&& pattern[..pattern.len() - 1]
.chars()
.rev()
.take_while(|character| *character == '\\')
.count()
.is_multiple_of(2);
if anchored_end {
pattern.pop();
}
let prefix = if anchored_start { "" } else { ".*" };
let suffix = if anchored_end { "" } else { ".*" };
let contains_pattern = format!("{prefix}({pattern}){suffix}");
Ok(Box::new(RegexQuery::from_pattern(
&contains_pattern,
fields.regex_text,
)?))
}
fn wildcard_query(pattern: &str, fields: SearchFields) -> Result<Box<dyn Query>, SearchError> {
Ok(Box::new(RegexQuery::from_pattern(
&wildcard_pattern(pattern),
src/search/src/search/indexer.rs
-41
View File
@@ -461,7 +461,6 @@ mod tests {
use crate::domain::{
AvailabilityStatus, ContentGroupBuilder, InfoHash, TorrentFile, TorrentRecord,
};
use crate::search::SearchMode;
use crate::storage::{RocksTorrentRepository, TorrentRepository};
use super::*;
@@ -674,32 +673,6 @@ mod tests {
assert_eq!(engine.search("1080p", 0, 10).unwrap().total, 1);
}
#[test]
fn regex_matches_names_and_file_paths_case_insensitively() {
let directory = TempDir::new().unwrap();
let engine = SearchEngine::open(directory.path()).unwrap();
index_records(&engine, &[record()]);
let name = engine
.search_with(SearchOptions {
query: r"ubuntu\s+linux\s+24\.0[0-9]".into(),
mode: SearchMode::Regex,
limit: 10,
..SearchOptions::default()
})
.unwrap();
assert_eq!(name.total, 1);
let path = engine
.search_with(SearchOptions {
query: r"ubuntu\.(iso|img)$".into(),
mode: SearchMode::Regex,
limit: 10,
..SearchOptions::default()
})
.unwrap();
assert_eq!(path.total, 1);
}
#[test]
fn wildcard_matches_names_and_file_extensions() {
let directory = TempDir::new().unwrap();
@@ -726,20 +699,6 @@ mod tests {
assert_eq!(missing.total, 0);
}
#[test]
fn invalid_regex_is_rejected() {
let directory = TempDir::new().unwrap();
let engine = SearchEngine::open(directory.path()).unwrap();
index_records(&engine, &[record()]);
let result = engine.search_with(SearchOptions {
query: "[".into(),
mode: SearchMode::Regex,
limit: 10,
..SearchOptions::default()
});
assert!(result.is_err());
}
#[test]
fn equal_content_is_collapsed_and_aliases_remain_searchable() {
let directory = TempDir::new().unwrap();
src/search/src/search/mod.rs
+1 -3
View File
@@ -10,9 +10,7 @@ mod schema;
pub(crate) use file_order::order_files;
pub use indexer::{SearchDiagnostics, SearchEngine};
pub use query::{
AvailabilitySummary, SearchHit, SearchMode, SearchOptions, SearchPage, SearchSort,
};
pub use query::{AvailabilitySummary, SearchHit, SearchOptions, SearchPage, SearchSort};
pub use runtime::{IndexRebuildReason, IndexState, IndexStatus, SearchRuntime};
pub(crate) use runtime::{RebuildSession, SearchBootstrap};
src/search/src/search/query.rs
-9
View File
@@ -17,18 +17,9 @@ pub enum SearchSort {
Discoveries,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SearchMode {
#[default]
Text,
Regex,
}
#[derive(Debug, Clone, Default)]
pub struct SearchOptions {
pub query: String,
pub mode: SearchMode,
pub offset: usize,
pub limit: usize,
pub min_size: Option<u64>,
src/web/README.md
+1 -1
View File
@@ -50,7 +50,7 @@ Axum 根据配置中的 `http.web_dir` 提供静态资源和单页回退 不需
## 已实现功能
- 关键词 文件名片段和精确 infohash 搜索
- 根据输入语法自动识别普通关键词 通配符和名称文件路径正则搜索
- 支持普通关键词 通配符和精确 infohash 搜索
- 相关度 时间 热度 大小和发现次数排序
- 有上限的结果分页和 URL 查询恢复
- 基于 reka-ui 的搜索结果数字分页和浏览器持久化每页数量选择
src/web/src/lib/api.ts
-4
View File
@@ -10,8 +10,6 @@ import type {
ServiceStats,
TorrentDetail,
} from '@/types/api'
import { isLikelyRegex } from '@/lib/search-mode'
interface SearchInput {
query: string
offset: number
@@ -45,13 +43,11 @@ export function search(input: SearchInput, signal?: AbortSignal): Promise<Search
limit: String(input.limit),
sort: input.sort,
})
if (isLikelyRegex(input.query)) params.set('mode', 'regex')
return request<SearchPage>(`/search?${params}`, signal)
}
export function getTorrent(infoHash: string, fileOffset = 0, fileLimit = 100, query = '', sort: SearchSort = 'latest', signal?: AbortSignal): Promise<TorrentDetail> {
const params = new URLSearchParams({ file_offset: String(fileOffset), file_limit: String(fileLimit), q: query.trim(), sort })
if (isLikelyRegex(query)) params.set('mode', 'regex')
return request<TorrentDetail>(`/torrents/${encodeURIComponent(infoHash)}?${params}`, signal)
}
src/web/src/lib/search-mode.ts
-17
View File
@@ -1,17 +0,0 @@
// 负责根据查询语法判断是否使用正则搜索
const REGEX_SIGNALS = [
/^\^/,
/(^|[^\\])\$$/,
/\\[AbBdDsSwWZz]/,
/(^|[^\\])\[[^\]]+\]/,
/(^|[^\\])\([^)]*\)/,
/(^|[^\\])\|/,
/(^|[^\\])\{(?:\d+|\d+,\d*|,\d+)\}/,
/\.\*|\.\+/,
]
export function isLikelyRegex(query: string): boolean {
const value = query.trim()
return value !== '' && REGEX_SIGNALS.some((signal) => signal.test(value))
}
src/web/src/pages/SearchPage.vue
+2 -2
View File
@@ -42,7 +42,7 @@ let indexTimer: number | null = null
const pageNumber = computed(() => Math.floor((page.value?.offset ?? 0) / limit.value) + 1)
const paginationTotal = computed(() => Math.min(page.value?.total ?? 0, 10_000 + limit.value))
const searchPlaceholder = '输入名称、文件名、*.iso、正则表达式或 40 位 infohash'
const searchPlaceholder = '输入名称、文件名、*.iso 或 40 位 infohash'
const resultPageSizeOptions = [
{ value: '10', label: '10 条/页' },
{ value: '20', label: '20 条/页' },
@@ -237,7 +237,7 @@ onBeforeUnmount(() => {
<div v-if="loading" class="flex min-h-72 items-center justify-center text-sm text-muted-foreground"><LoaderCircle class="mr-2 size-5 animate-spin" />正在搜索</div>
<div v-else-if="error" class="mt-4 flex min-h-72 flex-col items-center justify-center gap-4 rounded-xl border border-dashed text-center"><Server class="size-8 text-muted-foreground" /><div><p class="font-medium">无法完成搜索</p><p class="mt-1 text-sm text-muted-foreground">{{ error }}</p></div><Button variant="outline" @click="loadSearch">重试</Button></div>
<div v-else-if="page?.hits.length" class="mt-4 space-y-3"><SearchResultCard v-for="hit in page.hits" :key="hit.content_key" :hit="hit" @select="openDetail" /></div>
<div v-else class="mt-4 flex min-h-72 flex-col items-center justify-center rounded-xl border border-dashed text-center"><Search class="mb-3 size-8 text-muted-foreground" /><p class="font-medium">没有找到匹配内容</p><p class="mt-1 text-sm text-muted-foreground">尝试其他关键词、通配符或正则表达式</p></div>
<div v-else class="mt-4 flex min-h-72 flex-col items-center justify-center rounded-xl border border-dashed text-center"><Search class="mb-3 size-8 text-muted-foreground" /><p class="font-medium">没有找到匹配内容</p><p class="mt-1 text-sm text-muted-foreground">尝试其他关键词或通配符</p></div>
<div v-if="page && page.total > 0" class="mt-6 flex flex-wrap items-center justify-center gap-2">
<AppSelect :model-value="String(limit)" :options="resultPageSizeOptions" label="每页搜索结果数量" @update:model-value="changeResultPageSize" />