diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..42c3382 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +.gitignore +.tools +.run-* +*.log + +target +data +opencodes + +src/web/node_modules +src/web/dist +src/web/.vscode + +scripts diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..14b5537 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,107 @@ +# Docker 构建和运行 + +## 镜像结构 + +镜像使用三个构建阶段 + +- `oven/bun:1.3.14-debian` 负责编译 Web 静态资源 +- `rust:1.97.1-trixie` 负责构建 Rust 和 RocksDB +- `debian:trixie-slim` 只保留运行所需的二进制静态资源和 C++ 运行库 + +最终容器只运行一个非 root `dht-search` 进程 + +## 构建 + +在仓库根目录执行 + +```shell +docker build --platform linux/amd64 -t dht-search:dev . +``` + +构建会复用 Cargo registry Git 和 target BuildKit 缓存 + +可以使用以下命令检查镜像 + +```shell +docker image inspect dht-search:dev +docker run --rm --entrypoint id dht-search:dev +docker run --rm --entrypoint ldd dht-search:dev /dht-search/dht-search +``` + +## Compose 运行 + +在仓库根目录执行 + +```shell +docker compose up -d --build +docker compose logs -f +``` + +容器内应用统一位于 `/dht-search` + +```text +/dht-search/ +├── dht-search +├── config.toml +├── web/ +└── data/ +``` + +Compose 将仓库根目录的 `config.toml` 直接映射到 `/dht-search/config.toml` 网页保存配置时会同步修改宿主机文件 + +Compose 只创建一个 `dht-search-data` 命名卷并挂载到 `/dht-search/data` + +RocksDB Tantivy SQLite 日志和检查点全部位于该数据卷 + +配置保存通常使用临时文件原子替换 在 Docker 单文件挂载环境中会自动改用同步覆盖写入 + +本地配置保留 `127.0.0.1:8080` `src/web/dist` 和文件日志方便直接运行 容器启动参数会覆盖监听地址和静态目录并将日志切换到容器标准输出 + +HTTP 默认只发布到宿主机回环地址 UDP DHT 端口默认公开 如果端口冲突可以临时覆盖 + +```powershell +$env:DHT_HTTP_BIND = "127.0.0.1:18080" +$env:DHT_UDP_PORT = "22313" +docker compose up -d +``` + +## 验证 + +```shell +curl http://127.0.0.1:8080/health +curl http://127.0.0.1:8080/ready +docker compose logs -f +``` + +浏览器访问 `http://127.0.0.1:8080` + +公网访问 Web 应使用反向代理或 SSH 通道 + +## 停止和重启 + +```shell +docker compose stop +docker compose start +``` + +更新镜像或重新创建容器时执行 `docker compose up -d --build` 即可复用根配置和数据卷 + +`docker compose down` 只删除容器和网络 不删除数据卷 + +不要执行 `docker compose down --volumes` 除非已经确认 RocksDB 权威数据和其他运行数据都不再需要 + +## 远程设备 + +远程设备是 x86_64 时构建 `linux/amd64` + +需要导出镜像时执行 + +```shell +docker save dht-search:dev -o dht-search-dev.tar +``` + +将文件复制到远程设备后执行 + +```shell +docker load -i dht-search-dev.tar +``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..95e9856 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,70 @@ +# syntax=docker/dockerfile:1 + +FROM oven/bun:1.3.14-debian AS web-builder + +WORKDIR /build/src/web + +COPY src/web/package.json src/web/bun.lock ./ +RUN bun install --frozen-lockfile + +COPY src/web/ ./ +RUN bun run build + + +FROM rust:1.97.1-trixie AS rust-builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + clang \ + cmake \ + libclang-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +COPY Cargo.toml Cargo.lock ./ +COPY src/crawler ./src/crawler +COPY src/search ./src/search + +RUN --mount=type=cache,id=dht-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=dht-cargo-git,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=dht-cargo-target,target=/build/target,sharing=locked \ + cargo build --locked --release -p dht-search --bin dht-search \ + && install -Dm755 target/release/dht-search /out/dht-search + + +FROM debian:trixie-slim AS runtime + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libgcc-s1 \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --gid 10001 dht-search \ + && useradd \ + --uid 10001 \ + --gid 10001 \ + --no-create-home \ + --no-log-init \ + --shell /usr/sbin/nologin \ + dht-search \ + && install -d -o 10001 -g 10001 \ + /dht-search \ + /dht-search/data \ + /dht-search/web + +COPY --from=rust-builder /out/dht-search /dht-search/dht-search +COPY --from=web-builder /build/src/web/dist /dht-search/web +COPY --chmod=0644 --chown=10001:10001 config.toml /dht-search/config.toml + +USER 10001:10001 + +EXPOSE 8080/tcp +EXPOSE 12313/udp + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/dht-search/dht-search"] +CMD ["--config", "/dht-search/config.toml", "--http-listen", "0.0.0.0:8080", "--web-dir", "/dht-search/web", "--console-logging", "--no-file-logging"] diff --git a/TODOS.md b/TODOS.md index 85f4d5a..bd3dc5e 100644 --- a/TODOS.md +++ b/TODOS.md @@ -283,8 +283,8 @@ - [x] 完成一万十万和一百万条 release 基线并记录查询 P50 P95 P99 - [x] 记录每条元数据和每个索引文档的平均磁盘占用 - [x] 记录百万级基准进程峰值内存和索引总吞吐 -- [ ] 记录 RocksDB block cache memtable 和 compaction 指标 -- [ ] 记录 Tantivy IndexWriter 内存和 commit 延迟 +- [x] 记录 RocksDB block cache memtable 和 compaction 指标 +- [x] 记录 Tantivy IndexWriter 内存和 commit 延迟 - [ ] 根据实测调整批量大小队列容量和并发 - [x] 增加带排空阶段恢复滞回和探测失败保护的磁盘只读降级策略 - [x] 增加在线 RocksDB 检查点保留上限只读校验和带旧库保留的离线恢复 @@ -312,9 +312,10 @@ - [x] 固化 Linux 目标构建方式和 RocksDB 构建依赖 - [x] 生成 release 二进制并使用 SHA-256 校验部署 -- [ ] 定义配置数据日志和索引目录布局 -- [ ] 编写 systemd service -- [ ] 编写 systemd timer 支持间歇运行 +- [x] 定义 Docker 配置数据日志索引和静态资源目录布局 +- [x] 编写 Debian 三阶段最小运行镜像并使用非 root 用户完成构建启动和重启恢复测试 +- [x] 编写 Compose 配置管理端口数据卷重启策略和文件句柄限制 +- [x] Docker 将根目录唯一 `config.toml` 映射到容器并使用单独数据卷保存全部运行数据 - [x] 使用专用低权限 UID 运行验证 - [x] 固化 Xray 环境下的最小范围网络旁路 - [x] 验证高并发运行需要 `LimitNOFILE=65536` @@ -324,8 +325,8 @@ ### 验收标准 -- [ ] 新设备可以按文档完成部署 -- [ ] 服务重启不会丢失已提交数据 +- [ ] 新设备可以按 Docker 文档完成部署 +- [x] 容器重建并复用数据卷不会丢失已提交数据和诊断历史 - [ ] Xray 旁路只影响爬虫进程 - [ ] 更新失败时可以恢复上一版本二进制和数据 @@ -349,10 +350,9 @@ - [x] 精简诊断页即时指标并由趋势图承担重复的资源和吞吐数据 - [x] 使用页签拆分配置分类并隐藏底层配置存储位置 - [x] 将主配置和内容过滤规则合并为唯一 `config.toml` -- [ ] 将用户可编辑配置的持久化适配器从 TOML 迁移到 SQLite 完成二十四小时持续运行并继续观察私有内存 Metadata 成功率候选队列深度和每条成功 Metadata 的网络成本 RocksDB Tantivy HTTP 和进程资源指标已经接入诊断历史 后续根据长期实测继续调优 -下一轮结构优化优先拆分 crawler 中的运行统计调度器和抓取引擎以及 search 中的 RocksDB 适配器 不为拆分而新增 crate +下一步在公网 Linux 设备验证 Compose 容器网络重启策略数据卷权限和优雅停止 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2e868e1 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,28 @@ +name: dht-search + +services: + dht-search: + image: dht-search:dev + build: + context: . + dockerfile: Dockerfile + init: true + restart: unless-stopped + stop_grace_period: 120s + ports: + - "${DHT_HTTP_BIND:-127.0.0.1:8080}:8080/tcp" + - "${DHT_UDP_PORT:-12313}:12313/udp" + volumes: + - ./config.toml:/dht-search/config.toml + - dht-search-data:/dht-search/data + ulimits: + nofile: + soft: 65536 + hard: 65536 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + +volumes: + dht-search-data: diff --git a/src/search/src/config.rs b/src/search/src/config.rs index 00fa615..9f9ac37 100644 --- a/src/search/src/config.rs +++ b/src/search/src/config.rs @@ -6,6 +6,7 @@ mod service; mod store; use std::{ + net::SocketAddr, path::{Path, PathBuf}, sync::Arc, }; @@ -30,6 +31,14 @@ pub(crate) struct Cli { #[arg(long)] data_dir: Option, #[arg(long)] + http_listen: Option, + #[arg(long)] + web_dir: Option, + #[arg(long)] + console_logging: bool, + #[arg(long)] + no_file_logging: bool, + #[arg(long)] run_duration_secs: Option, #[arg(long)] restore_checkpoint: Option, @@ -54,6 +63,22 @@ impl Cli { dto.data_dir = data_dir; command_line_overrides.push("data_dir".to_owned()); } + if let Some(http_listen) = self.http_listen { + dto.http.listen = http_listen; + command_line_overrides.push("http.listen".to_owned()); + } + if let Some(web_dir) = self.web_dir { + dto.http.web_dir = web_dir; + command_line_overrides.push("http.web_dir".to_owned()); + } + if self.console_logging { + dto.logging.console_enabled = true; + command_line_overrides.push("logging.console_enabled".to_owned()); + } + if self.no_file_logging { + dto.logging.file_enabled = false; + command_line_overrides.push("logging.file_enabled".to_owned()); + } if self.run_duration_secs.is_some() { dto.run_duration_secs = self.run_duration_secs; command_line_overrides.push("run_duration_secs".to_owned()); @@ -139,6 +164,10 @@ mod tests { let startup = Cli { config: config_path, data_dir: None, + http_listen: None, + web_dir: None, + console_logging: false, + no_file_logging: false, run_duration_secs: None, restore_checkpoint: None, } @@ -166,6 +195,10 @@ mod tests { let error = Cli { config: config_path, data_dir: None, + http_listen: None, + web_dir: None, + console_logging: false, + no_file_logging: false, run_duration_secs: None, restore_checkpoint: None, } @@ -174,6 +207,44 @@ mod tests { assert!(matches!(error, AppError::Toml(_))); } + #[test] + fn command_line_can_override_container_runtime_settings() { + let directory = TempDir::new().unwrap(); + let config_path = directory.path().join("service.toml"); + fs::write( + &config_path, + "[http]\nlisten = '127.0.0.1:8080'\nweb_dir = 'src/web/dist'", + ) + .unwrap(); + let listen = "0.0.0.0:8080".parse().unwrap(); + let startup = Cli { + config: config_path, + data_dir: None, + http_listen: Some(listen), + web_dir: Some(PathBuf::from("/dht-search/web")), + console_logging: true, + no_file_logging: true, + run_duration_secs: None, + restore_checkpoint: None, + } + .load() + .unwrap(); + + assert_eq!(startup.app.http.listen, listen); + assert!(startup.app.http.web_dir.ends_with("dht-search/web")); + assert_eq!( + startup.config_service.snapshot().command_line_overrides, + [ + "http.listen", + "http.web_dir", + "logging.console_enabled", + "logging.file_enabled" + ] + ); + assert!(startup.app.logging.console_enabled); + assert!(!startup.app.logging.file_enabled); + } + #[test] fn zero_metadata_limit_is_rejected() { let mut dto = AppConfigDto::default(); diff --git a/src/search/src/config/store.rs b/src/search/src/config/store.rs index 31772e2..f8d822a 100644 --- a/src/search/src/config/store.rs +++ b/src/search/src/config/store.rs @@ -1,4 +1,4 @@ -// 负责从持久化介质读取并原子保存用户配置 DTO +// 负责从持久化介质读取保存用户配置 DTO 并兼容 Docker 单文件挂载 use std::{ fs::{self, OpenOptions}, @@ -55,7 +55,14 @@ impl ConfigStore for TomlConfigStore { file.write_all(contents.as_bytes())?; file.sync_all()?; drop(file); - replace_file(&temporary, &self.path)?; + match replace_file(&temporary, &self.path) { + Ok(()) => {} + Err(error) if is_bind_mount_replace_error(&error) => { + write_in_place(&self.path, contents.as_bytes())?; + fs::remove_file(&temporary)?; + } + Err(error) => return Err(error), + } sync_parent(parent)?; Ok::<(), std::io::Error>(()) })(); @@ -70,6 +77,25 @@ impl ConfigStore for TomlConfigStore { } } +fn write_in_place(destination: &Path, contents: &[u8]) -> std::io::Result<()> { + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(destination)?; + file.write_all(contents)?; + file.sync_all() +} + +#[cfg(target_os = "linux")] +fn is_bind_mount_replace_error(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(16) +} + +#[cfg(not(target_os = "linux"))] +fn is_bind_mount_replace_error(_error: &std::io::Error) -> bool { + false +} + fn temporary_path(destination: &Path) -> PathBuf { let sequence = TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed); let file_name = destination @@ -152,4 +178,15 @@ mod tests { 0 ); } + + #[test] + fn in_place_fallback_truncates_previous_contents() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("service.toml"); + fs::write(&path, "old contents that are longer").unwrap(); + + write_in_place(&path, b"new").unwrap(); + + assert_eq!(fs::read_to_string(path).unwrap(), "new"); + } } diff --git a/src/web/src/env.d.ts b/src/web/src/env.d.ts new file mode 100644 index 0000000..ab333dd --- /dev/null +++ b/src/web/src/env.d.ts @@ -0,0 +1,8 @@ +/// + +declare module '*.vue' { + import type { DefineComponent } from 'vue' + + const component: DefineComponent, Record, unknown> + export default component +}