mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
a4711b88dd
## Why `fs/readFile` buffers the entire file in one response, which makes large remote reads expensive and prevents callers from applying backpressure. We need an opt-in streaming path with bounded block sizes while preserving the existing single-call API for small and sandboxed reads. ## What changed - Add `ExecServerClient::stream`, returning a named `FileReadStream` that implements `futures::Stream` and yields immutable 1 MiB byte blocks. - Add internal `fs/open`, `fs/readBlock`, and `fs/close` RPCs. `fs/readBlock` accepts an explicit offset and length. - Keep unsandboxed files open between block reads, cap open handles per connection, and clean them up on EOF, error, stream drop, explicit close, or connection shutdown. - Reject platform-sandboxed streaming opens instead of turning the one-shot sandbox helper into a persistent server. Existing `fs/readFile` behavior is unchanged. ## Testing - `just test -p codex-exec-server` - Integration coverage for 1 MiB chunking, exact block-boundary EOF, sandbox rejection, and continued reads from the opened file after path replacement. - Handle-manager coverage for non-sequential offsets, variable block lengths, the 128-handle limit, and capacity release after close.
379 lines
11 KiB
Rust
379 lines
11 KiB
Rust
use base64::Engine as _;
|
|
use base64::engine::general_purpose::STANDARD;
|
|
use codex_app_server_protocol::JSONRPCErrorError;
|
|
use codex_utils_path_uri::PathUri;
|
|
use tokio::io;
|
|
|
|
use crate::CopyOptions;
|
|
use crate::CreateDirectoryOptions;
|
|
use crate::ExecServerRuntimePaths;
|
|
use crate::ExecutorFileSystem;
|
|
use crate::ExecutorFileSystemFuture;
|
|
use crate::FileMetadata;
|
|
use crate::FileSystemReadStream;
|
|
use crate::FileSystemResult;
|
|
use crate::FileSystemSandboxContext;
|
|
use crate::ReadDirectoryEntry;
|
|
use crate::RemoveOptions;
|
|
use crate::fs_helper::FsHelperPayload;
|
|
use crate::fs_helper::FsHelperRequest;
|
|
use crate::fs_sandbox::FileSystemSandboxRunner;
|
|
use crate::protocol::FsCanonicalizeParams;
|
|
use crate::protocol::FsCopyParams;
|
|
use crate::protocol::FsCreateDirectoryParams;
|
|
use crate::protocol::FsGetMetadataParams;
|
|
use crate::protocol::FsReadDirectoryParams;
|
|
use crate::protocol::FsReadFileParams;
|
|
use crate::protocol::FsRemoveParams;
|
|
use crate::protocol::FsWriteFileParams;
|
|
|
|
#[derive(Clone)]
|
|
pub struct SandboxedFileSystem {
|
|
sandbox_runner: FileSystemSandboxRunner,
|
|
}
|
|
|
|
impl SandboxedFileSystem {
|
|
pub fn new(runtime_paths: ExecServerRuntimePaths) -> Self {
|
|
Self {
|
|
sandbox_runner: FileSystemSandboxRunner::new(runtime_paths),
|
|
}
|
|
}
|
|
|
|
async fn run_sandboxed(
|
|
&self,
|
|
sandbox: &FileSystemSandboxContext,
|
|
request: FsHelperRequest,
|
|
) -> FileSystemResult<FsHelperPayload> {
|
|
self.sandbox_runner
|
|
.run(sandbox, request)
|
|
.await
|
|
.map_err(map_sandbox_error)
|
|
}
|
|
}
|
|
|
|
impl SandboxedFileSystem {
|
|
async fn canonicalize(
|
|
&self,
|
|
path: &PathUri,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<PathUri> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
let response = self
|
|
.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::Canonicalize(FsCanonicalizeParams {
|
|
path: path.clone(),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_canonicalize()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(response.path)
|
|
}
|
|
|
|
async fn read_file(
|
|
&self,
|
|
path: &PathUri,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<Vec<u8>> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
let response = self
|
|
.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::ReadFile(FsReadFileParams {
|
|
path: path.clone(),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_read_file()
|
|
.map_err(map_sandbox_error)?;
|
|
STANDARD.decode(response.data_base64).map_err(|err| {
|
|
io::Error::new(
|
|
io::ErrorKind::InvalidData,
|
|
format!("fs/readFile returned invalid base64 dataBase64: {err}"),
|
|
)
|
|
})
|
|
}
|
|
|
|
async fn write_file(
|
|
&self,
|
|
path: &PathUri,
|
|
contents: Vec<u8>,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<()> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
self.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::WriteFile(FsWriteFileParams {
|
|
path: path.clone(),
|
|
data_base64: STANDARD.encode(contents),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_write_file()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn create_directory(
|
|
&self,
|
|
path: &PathUri,
|
|
options: CreateDirectoryOptions,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<()> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
self.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::CreateDirectory(FsCreateDirectoryParams {
|
|
path: path.clone(),
|
|
recursive: Some(options.recursive),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_create_directory()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn get_metadata(
|
|
&self,
|
|
path: &PathUri,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<FileMetadata> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
let response = self
|
|
.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::GetMetadata(FsGetMetadataParams {
|
|
path: path.clone(),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_get_metadata()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(FileMetadata {
|
|
is_directory: response.is_directory,
|
|
is_file: response.is_file,
|
|
is_symlink: response.is_symlink,
|
|
size: response.size,
|
|
created_at_ms: response.created_at_ms,
|
|
modified_at_ms: response.modified_at_ms,
|
|
})
|
|
}
|
|
|
|
async fn read_directory(
|
|
&self,
|
|
path: &PathUri,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<Vec<ReadDirectoryEntry>> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
let response = self
|
|
.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::ReadDirectory(FsReadDirectoryParams {
|
|
path: path.clone(),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_read_directory()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(response
|
|
.entries
|
|
.into_iter()
|
|
.map(|entry| ReadDirectoryEntry {
|
|
file_name: entry.file_name,
|
|
is_directory: entry.is_directory,
|
|
is_file: entry.is_file,
|
|
})
|
|
.collect())
|
|
}
|
|
|
|
async fn remove(
|
|
&self,
|
|
path: &PathUri,
|
|
remove_options: RemoveOptions,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<()> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(path)?;
|
|
self.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::Remove(FsRemoveParams {
|
|
path: path.clone(),
|
|
recursive: Some(remove_options.recursive),
|
|
force: Some(remove_options.force),
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_remove()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn copy(
|
|
&self,
|
|
source_path: &PathUri,
|
|
destination_path: &PathUri,
|
|
options: CopyOptions,
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<()> {
|
|
let sandbox = require_platform_sandbox(sandbox)?;
|
|
validate_native_path(source_path)?;
|
|
validate_native_path(destination_path)?;
|
|
self.run_sandboxed(
|
|
sandbox,
|
|
FsHelperRequest::Copy(FsCopyParams {
|
|
source_path: source_path.clone(),
|
|
destination_path: destination_path.clone(),
|
|
recursive: options.recursive,
|
|
sandbox: None,
|
|
}),
|
|
)
|
|
.await?
|
|
.expect_copy()
|
|
.map_err(map_sandbox_error)?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl ExecutorFileSystem for SandboxedFileSystem {
|
|
fn canonicalize<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, PathUri> {
|
|
Box::pin(SandboxedFileSystem::canonicalize(self, path, sandbox))
|
|
}
|
|
|
|
fn read_file<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, Vec<u8>> {
|
|
Box::pin(SandboxedFileSystem::read_file(self, path, sandbox))
|
|
}
|
|
|
|
fn read_file_stream<'a>(
|
|
&'a self,
|
|
_path: &'a PathUri,
|
|
_sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> {
|
|
Box::pin(async {
|
|
Err(io::Error::new(
|
|
io::ErrorKind::Unsupported,
|
|
"streaming file reads do not support platform sandboxing",
|
|
))
|
|
})
|
|
}
|
|
|
|
fn write_file<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
contents: Vec<u8>,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(SandboxedFileSystem::write_file(
|
|
self, path, contents, sandbox,
|
|
))
|
|
}
|
|
|
|
fn create_directory<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
options: CreateDirectoryOptions,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(SandboxedFileSystem::create_directory(
|
|
self, path, options, sandbox,
|
|
))
|
|
}
|
|
|
|
fn get_metadata<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, FileMetadata> {
|
|
Box::pin(SandboxedFileSystem::get_metadata(self, path, sandbox))
|
|
}
|
|
|
|
fn read_directory<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, Vec<ReadDirectoryEntry>> {
|
|
Box::pin(SandboxedFileSystem::read_directory(self, path, sandbox))
|
|
}
|
|
|
|
fn remove<'a>(
|
|
&'a self,
|
|
path: &'a PathUri,
|
|
remove_options: RemoveOptions,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(SandboxedFileSystem::remove(
|
|
self,
|
|
path,
|
|
remove_options,
|
|
sandbox,
|
|
))
|
|
}
|
|
|
|
fn copy<'a>(
|
|
&'a self,
|
|
source_path: &'a PathUri,
|
|
destination_path: &'a PathUri,
|
|
options: CopyOptions,
|
|
sandbox: Option<&'a FileSystemSandboxContext>,
|
|
) -> ExecutorFileSystemFuture<'a, ()> {
|
|
Box::pin(SandboxedFileSystem::copy(
|
|
self,
|
|
source_path,
|
|
destination_path,
|
|
options,
|
|
sandbox,
|
|
))
|
|
}
|
|
}
|
|
|
|
fn validate_native_path(path: &PathUri) -> FileSystemResult<()> {
|
|
path.to_abs_path().map(drop)
|
|
}
|
|
|
|
fn require_platform_sandbox(
|
|
sandbox: Option<&FileSystemSandboxContext>,
|
|
) -> FileSystemResult<&FileSystemSandboxContext> {
|
|
sandbox
|
|
.filter(|sandbox| sandbox.should_run_in_sandbox())
|
|
.ok_or_else(|| {
|
|
io::Error::new(
|
|
io::ErrorKind::InvalidInput,
|
|
"sandboxed filesystem operations require ReadOnly or WorkspaceWrite sandbox policy",
|
|
)
|
|
})
|
|
}
|
|
|
|
fn map_sandbox_error(error: JSONRPCErrorError) -> io::Error {
|
|
match error.code {
|
|
-32004 => io::Error::new(io::ErrorKind::NotFound, error.message),
|
|
-32600 => io::Error::new(io::ErrorKind::InvalidInput, error.message),
|
|
_ => io::Error::other(error.message),
|
|
}
|
|
}
|
|
|
|
#[cfg(all(test, any(unix, windows)))]
|
|
#[path = "sandboxed_file_system_path_uri_tests.rs"]
|
|
mod path_uri_tests;
|