mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
feat: support local refs and defs in tool input schemas (#23357)
# Why
Some connector tool input schemas use local JSON Schema references and
definition tables to avoid duplicating large nested shapes. Codex
previously lowered these schemas into the supported subset in a way that
could discard `$ref`-only schema objects and lose the corresponding
definitions, which made non-strict tool registration less faithful than
the original connector schema.
This keeps the existing minimal-lowering policy: Codex still does not
raw-pass through arbitrary JSON Schema, but it now preserves local
reference structure that fits the Responses-compatible subset and prunes
definition entries that cannot be reached by following `$ref`s from the
root schema after sanitization, including refs found transitively inside
other reachable definitions. The pruning matters because Responses
parses definition tables even when entries are unused, so keeping dead
definitions wastes prompt tokens.
# What changed
- Added `$ref`, `$defs`, and legacy `definitions` fields to the tool
`JsonSchema` representation.
- Updated `parse_tool_input_schema` lowering so `$ref`-only schema
objects survive sanitization instead of becoming `{}`.
- Sanitized definition tables recursively and dropped malformed
definition tables so non-strict registration degrades gracefully.
- Added reachability pruning for root definition tables by starting from
refs outside definition tables, then following refs inside reachable
definitions.
- Added JSON Pointer decoding for local definition refs such as
`#/$defs/Foo~1Bar`.
# Verification
ran local golden-schema probes against representative connector schemas
to validate behavior on real generated schemas:
| Golden schema | Before bytes | After bytes | `$defs` before -> after |
`$ref` before -> after | Result |
|---|---:|---:|---:|---:|---|
| `google_calendar/create_space` | 7111 | 4526 | 7 -> 7 | 7 -> 7 | all
definitions preserved because all are reachable |
| `figma/apply_file_variable_changes` | 4609 | 999 | 8 -> 5 | 8 -> 5 |
unused defs pruned after unsupported `oneOf` shapes lower away |
| `snowflake/list_catalog_integrations` | 1380 | 404 | 3 -> 0 | 0 -> 0 |
all defs pruned because none are referenced |
| `dropbox/create_shared_link` | 8894 | 1836 | 14 -> 4 | 9 -> 4 | only
defs reachable from the root schema after sanitization are retained,
including transitively through other retained defs |
Token increase across golden schema due to this change:
<img width="817" height="366" alt="Screenshot 2026-05-19 at 1 47 04 PM"
src="https://github.com/user-attachments/assets/d5c80fe9-da85-41e6-8ac7-a01d1e0b0f71"
/>
This commit is contained in:
Celia Chen
committed by
GitHub
Unverified
GitHub
parent
5a6e905994
commit
0cec508148
Generated
+1
@@ -1150,6 +1150,7 @@
|
||||
"jni_0.21.1": "{\"dependencies\":[{\"name\":\"cesu8\",\"req\":\"^1.1.0\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"combine\",\"req\":\"^4.1.0\"},{\"name\":\"java-locator\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"jni-sys\",\"req\":\"^0.3.0\"},{\"name\":\"libloading\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"name\":\"thiserror\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rusty-fork\",\"req\":\"^0.3.0\"},{\"kind\":\"build\",\"name\":\"walkdir\",\"req\":\"^2\"},{\"features\":[\"Win32_Globalization\"],\"name\":\"windows-sys\",\"req\":\"^0.45.0\",\"target\":\"cfg(windows)\"},{\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.13.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"invocation\":[\"java-locator\",\"libloading\"]}}",
|
||||
"jobserver_0.1.34": "{\"dependencies\":[{\"features\":[\"std\"],\"name\":\"getrandom\",\"req\":\"^0.3.2\",\"target\":\"cfg(windows)\"},{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"fs\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.28.0\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10.1\"}],\"features\":{}}",
|
||||
"js-sys_0.3.85": "{\"dependencies\":[{\"default_features\":false,\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.108\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"wasm-bindgen/std\"]}}",
|
||||
"jsonptr_0.7.1": "{\"dependencies\":[{\"features\":[\"fancy\"],\"name\":\"miette\",\"optional\":true,\"req\":\"^7.4.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0.0\"},{\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.203\"},{\"features\":[\"alloc\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.119\"},{\"name\":\"syn\",\"optional\":true,\"req\":\"^1.0.109\",\"target\":\"cfg(any())\"},{\"name\":\"toml\",\"optional\":true,\"req\":\"^0.8\"}],\"features\":{\"assign\":[],\"default\":[\"std\",\"serde\",\"json\",\"resolve\",\"assign\",\"delete\"],\"delete\":[\"resolve\"],\"json\":[\"dep:serde_json\",\"serde\"],\"miette\":[\"dep:miette\",\"std\"],\"resolve\":[],\"std\":[\"serde/std\",\"serde_json?/std\"],\"toml\":[\"dep:toml\",\"serde\",\"std\"]}}",
|
||||
"jsonwebtoken_9.3.1": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\")))))\"},{\"name\":\"js-sys\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"pem\",\"optional\":true,\"req\":\"^3\"},{\"features\":[\"std\"],\"name\":\"ring\",\"req\":\"^0.17.4\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"std\",\"wasm32_unknown_unknown_js\"],\"name\":\"ring\",\"req\":\"^0.17.4\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"simple_asn1\",\"optional\":true,\"req\":\"^0.6\"},{\"features\":[\"wasm-bindgen\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\")))))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.1\"}],\"features\":{\"default\":[\"use_pem\"],\"use_pem\":[\"pem\",\"simple_asn1\"]}}",
|
||||
"keyring_3.6.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.2\",\"target\":\"cfg(target_os = \\\"windows\\\")\"},{\"features\":[\"derive\",\"wrap_help\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4\"},{\"name\":\"dbus-secret-service\",\"optional\":true,\"req\":\"^4.0.0-rc.1\",\"target\":\"cfg(target_os = \\\"openbsd\\\")\"},{\"name\":\"dbus-secret-service\",\"optional\":true,\"req\":\"^4.0.0-rc.2\",\"target\":\"cfg(target_os = \\\"linux\\\")\"},{\"name\":\"dbus-secret-service\",\"optional\":true,\"req\":\"^4.0.1\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11.5\"},{\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"features\":[\"std\"],\"name\":\"linux-keyutils\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(target_os = \\\"linux\\\")\"},{\"name\":\"log\",\"req\":\"^0.4.22\"},{\"name\":\"openssl\",\"optional\":true,\"req\":\"^0.10.66\"},{\"kind\":\"dev\",\"name\":\"rpassword\",\"req\":\"^7\"},{\"kind\":\"dev\",\"name\":\"rprompt\",\"req\":\"^2\"},{\"name\":\"secret-service\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"name\":\"secret-service\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"linux\\\")\"},{\"name\":\"secret-service\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"openbsd\\\")\"},{\"name\":\"security-framework\",\"optional\":true,\"req\":\"^2\",\"target\":\"cfg(target_os = \\\"ios\\\")\"},{\"name\":\"security-framework\",\"optional\":true,\"req\":\"^3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"kind\":\"dev\",\"name\":\"whoami\",\"req\":\"^1.5\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Credentials\"],\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.60\",\"target\":\"cfg(target_os = \\\"windows\\\")\"},{\"name\":\"zbus\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"name\":\"zbus\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"linux\\\")\"},{\"name\":\"zbus\",\"optional\":true,\"req\":\"^4\",\"target\":\"cfg(target_os = \\\"openbsd\\\")\"},{\"name\":\"zeroize\",\"req\":\"^1.8.1\",\"target\":\"cfg(target_os = \\\"windows\\\")\"}],\"features\":{\"apple-native\":[\"dep:security-framework\"],\"async-io\":[\"zbus?/async-io\"],\"async-secret-service\":[\"dep:secret-service\",\"dep:zbus\"],\"crypto-openssl\":[\"dbus-secret-service?/crypto-openssl\",\"secret-service?/crypto-openssl\"],\"crypto-rust\":[\"dbus-secret-service?/crypto-rust\",\"secret-service?/crypto-rust\"],\"linux-native\":[\"dep:linux-keyutils\"],\"linux-native-async-persistent\":[\"linux-native\",\"async-secret-service\"],\"linux-native-sync-persistent\":[\"linux-native\",\"sync-secret-service\"],\"sync-secret-service\":[\"dep:dbus-secret-service\"],\"tokio\":[\"zbus?/tokio\"],\"vendored\":[\"dbus-secret-service?/vendored\",\"openssl?/vendored\"],\"windows-native\":[\"dep:windows-sys\",\"dep:byteorder\"]}}",
|
||||
"kqueue-sys_1.0.4": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^1.2.1\"},{\"name\":\"libc\",\"req\":\"^0.2.74\"}],\"features\":{}}",
|
||||
|
||||
Generated
+8
@@ -3773,12 +3773,14 @@ dependencies = [
|
||||
"codex-utils-output-truncation",
|
||||
"codex-utils-pty",
|
||||
"codex-utils-string",
|
||||
"jsonptr",
|
||||
"pretty_assertions",
|
||||
"rmcp",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8125,6 +8127,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonptr"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe"
|
||||
|
||||
[[package]]
|
||||
name = "jsonwebtoken"
|
||||
version = "9.3.1"
|
||||
|
||||
@@ -301,6 +301,7 @@ indexmap = "2.12.0"
|
||||
insta = "1.46.3"
|
||||
inventory = "0.3.19"
|
||||
itertools = "0.14.0"
|
||||
jsonptr = { version = "0.7.1", default-features = false }
|
||||
jsonwebtoken = "9.3.1"
|
||||
keyring = { version = "3.6", default-features = false }
|
||||
landlock = "0.4.4"
|
||||
|
||||
@@ -17,6 +17,7 @@ codex-utils-absolute-path = { workspace = true }
|
||||
codex-utils-output-truncation = { workspace = true }
|
||||
codex-utils-pty = { workspace = true }
|
||||
codex-utils-string = { workspace = true }
|
||||
jsonptr = { workspace = true }
|
||||
rmcp = { workspace = true, default-features = false, features = [
|
||||
"base64",
|
||||
"macros",
|
||||
@@ -27,6 +28,7 @@ serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
|
||||
@@ -3,6 +3,10 @@ use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
const DEFINITION_TABLE_KEYS: [&str; 2] = ["$defs", "definitions"];
|
||||
const SCHEMA_CHILD_KEYS: [&str; 2] = ["items", "anyOf"];
|
||||
|
||||
/// Primitive JSON Schema type names we support in tool definitions.
|
||||
///
|
||||
@@ -33,6 +37,8 @@ pub enum JsonSchemaType {
|
||||
/// Generic JSON-Schema subset needed for our tool definitions.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct JsonSchema {
|
||||
#[serde(rename = "$ref", skip_serializing_if = "Option::is_none")]
|
||||
pub schema_ref: Option<String>,
|
||||
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
|
||||
pub schema_type: Option<JsonSchemaType>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -52,6 +58,10 @@ pub struct JsonSchema {
|
||||
pub additional_properties: Option<AdditionalProperties>,
|
||||
#[serde(rename = "anyOf", skip_serializing_if = "Option::is_none")]
|
||||
pub any_of: Option<Vec<JsonSchema>>,
|
||||
#[serde(rename = "$defs", skip_serializing_if = "Option::is_none")]
|
||||
pub defs: Option<BTreeMap<String, JsonSchema>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub definitions: Option<BTreeMap<String, JsonSchema>>,
|
||||
}
|
||||
|
||||
impl JsonSchema {
|
||||
@@ -149,6 +159,7 @@ impl From<JsonSchema> for AdditionalProperties {
|
||||
pub fn parse_tool_input_schema(input_schema: &JsonValue) -> Result<JsonSchema, serde_json::Error> {
|
||||
let mut input_schema = input_schema.clone();
|
||||
sanitize_json_schema(&mut input_schema);
|
||||
prune_unreachable_definitions(&mut input_schema);
|
||||
let schema: JsonSchema = serde_json::from_value(input_schema)?;
|
||||
if matches!(
|
||||
schema.schema_type,
|
||||
@@ -159,10 +170,55 @@ pub fn parse_tool_input_schema(input_schema: &JsonValue) -> Result<JsonSchema, s
|
||||
Ok(schema)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DefinitionTraversal {
|
||||
Include,
|
||||
Skip,
|
||||
}
|
||||
|
||||
fn for_each_schema_child(
|
||||
map: &serde_json::Map<String, JsonValue>,
|
||||
definition_traversal: DefinitionTraversal,
|
||||
visitor: &mut impl FnMut(&JsonValue),
|
||||
) {
|
||||
if let Some(properties) = map.get("properties")
|
||||
&& let Some(properties_map) = properties.as_object()
|
||||
{
|
||||
for value in properties_map.values() {
|
||||
visitor(value);
|
||||
}
|
||||
}
|
||||
|
||||
for key in SCHEMA_CHILD_KEYS {
|
||||
if let Some(value) = map.get(key) {
|
||||
visitor(value);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(additional_properties) = map.get("additionalProperties")
|
||||
&& !matches!(additional_properties, JsonValue::Bool(_))
|
||||
{
|
||||
visitor(additional_properties);
|
||||
}
|
||||
|
||||
if definition_traversal == DefinitionTraversal::Include {
|
||||
for key in DEFINITION_TABLE_KEYS {
|
||||
if let Some(definitions) = map.get(key)
|
||||
&& let Some(definitions_map) = definitions.as_object()
|
||||
{
|
||||
for value in definitions_map.values() {
|
||||
visitor(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize a JSON Schema (as serde_json::Value) so it can fit our limited
|
||||
/// schema representation. This function:
|
||||
/// - Ensures every typed schema object has a `"type"` when required.
|
||||
/// - Preserves explicit `anyOf`.
|
||||
/// - Preserves `$ref` and reachable local `$defs` / `definitions`.
|
||||
/// - Collapses `const` into single-value `enum`.
|
||||
/// - Fills required child fields for object/array schema types, including
|
||||
/// nullable unions, with permissive defaults when absent.
|
||||
@@ -200,6 +256,9 @@ fn sanitize_json_schema(value: &mut JsonValue) {
|
||||
if let Some(value) = map.get_mut("anyOf") {
|
||||
sanitize_json_schema(value);
|
||||
}
|
||||
for table in DEFINITION_TABLE_KEYS {
|
||||
sanitize_schema_table(map, table);
|
||||
}
|
||||
|
||||
if let Some(const_value) = map.remove("const") {
|
||||
map.insert("enum".to_string(), JsonValue::Array(vec![const_value]));
|
||||
@@ -207,7 +266,7 @@ fn sanitize_json_schema(value: &mut JsonValue) {
|
||||
|
||||
let mut schema_types = normalized_schema_types(map);
|
||||
|
||||
if schema_types.is_empty() && map.contains_key("anyOf") {
|
||||
if schema_types.is_empty() && (map.contains_key("$ref") || map.contains_key("anyOf")) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -241,6 +300,29 @@ fn sanitize_json_schema(value: &mut JsonValue) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize a schema definition table before deserializing into `JsonSchema`.
|
||||
///
|
||||
/// Definition tables must be objects. Codex keeps valid definition tables and
|
||||
/// recursively applies the same compatibility lowering used for inline schemas,
|
||||
/// but drops malformed tables so `strict: false` tool registration degrades
|
||||
/// gracefully instead of failing on an unreachable or invalid definition table.
|
||||
fn sanitize_schema_table(map: &mut serde_json::Map<String, JsonValue>, key: &str) {
|
||||
let should_remove = match map.get_mut(key) {
|
||||
Some(JsonValue::Object(definitions)) => {
|
||||
for definition in definitions.values_mut() {
|
||||
sanitize_json_schema(definition);
|
||||
}
|
||||
false
|
||||
}
|
||||
Some(_) => true,
|
||||
None => false,
|
||||
};
|
||||
|
||||
if should_remove {
|
||||
map.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_default_children_for_schema_types(
|
||||
map: &mut serde_json::Map<String, JsonValue>,
|
||||
schema_types: &[JsonSchemaPrimitiveType],
|
||||
@@ -257,6 +339,143 @@ fn ensure_default_children_for_schema_types(
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct DefinitionPointer {
|
||||
table: &'static str,
|
||||
name: String,
|
||||
}
|
||||
|
||||
/// Prune unused root definition entries to avoid sending tokens for definitions
|
||||
/// the tool schema never references.
|
||||
fn prune_unreachable_definitions(value: &mut JsonValue) {
|
||||
let reachable = collect_reachable_definitions(value);
|
||||
let JsonValue::Object(map) = value else {
|
||||
return;
|
||||
};
|
||||
|
||||
for table in DEFINITION_TABLE_KEYS {
|
||||
prune_schema_table(map, table, &reachable);
|
||||
}
|
||||
}
|
||||
|
||||
fn prune_schema_table(
|
||||
map: &mut serde_json::Map<String, JsonValue>,
|
||||
table: &'static str,
|
||||
reachable: &BTreeSet<DefinitionPointer>,
|
||||
) {
|
||||
let Some(JsonValue::Object(definitions)) = map.get_mut(table) else {
|
||||
return;
|
||||
};
|
||||
|
||||
definitions.retain(|name, _| {
|
||||
reachable.contains(&DefinitionPointer {
|
||||
table,
|
||||
name: name.clone(),
|
||||
})
|
||||
});
|
||||
|
||||
if definitions.is_empty() {
|
||||
map.remove(table);
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_reachable_definitions(value: &JsonValue) -> BTreeSet<DefinitionPointer> {
|
||||
let mut reachable = BTreeSet::new();
|
||||
let mut pending = Vec::new();
|
||||
|
||||
collect_refs_outside_definitions(value, &mut pending);
|
||||
|
||||
while let Some(pointer) = pending.pop() {
|
||||
if !reachable.insert(pointer.clone()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(definition) = definition_for_pointer(value, &pointer) {
|
||||
collect_refs(definition, &mut pending);
|
||||
}
|
||||
}
|
||||
|
||||
reachable
|
||||
}
|
||||
|
||||
fn collect_refs_outside_definitions(value: &JsonValue, refs: &mut Vec<DefinitionPointer>) {
|
||||
match value {
|
||||
JsonValue::Array(values) => {
|
||||
for value in values {
|
||||
collect_refs_outside_definitions(value, refs);
|
||||
}
|
||||
}
|
||||
JsonValue::Object(map) => {
|
||||
collect_ref_from_map(map, refs);
|
||||
for_each_schema_child(map, DefinitionTraversal::Skip, &mut |value| {
|
||||
collect_refs_outside_definitions(value, refs);
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_refs(value: &JsonValue, refs: &mut Vec<DefinitionPointer>) {
|
||||
match value {
|
||||
JsonValue::Array(values) => {
|
||||
for value in values {
|
||||
collect_refs(value, refs);
|
||||
}
|
||||
}
|
||||
JsonValue::Object(map) => {
|
||||
collect_ref_from_map(map, refs);
|
||||
for value in map.values() {
|
||||
collect_refs(value, refs);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_ref_from_map(
|
||||
map: &serde_json::Map<String, JsonValue>,
|
||||
refs: &mut Vec<DefinitionPointer>,
|
||||
) {
|
||||
if let Some(JsonValue::String(schema_ref)) = map.get("$ref")
|
||||
&& let Some(pointer) = parse_local_definition_ref(schema_ref)
|
||||
{
|
||||
refs.push(pointer);
|
||||
}
|
||||
}
|
||||
|
||||
fn definition_for_pointer<'a>(
|
||||
value: &'a JsonValue,
|
||||
pointer: &DefinitionPointer,
|
||||
) -> Option<&'a JsonValue> {
|
||||
let JsonValue::Object(map) = value else {
|
||||
return None;
|
||||
};
|
||||
|
||||
map.get(pointer.table)
|
||||
.and_then(JsonValue::as_object)
|
||||
.and_then(|definitions| definitions.get(&pointer.name))
|
||||
}
|
||||
|
||||
fn parse_local_definition_ref(schema_ref: &str) -> Option<DefinitionPointer> {
|
||||
let fragment = schema_ref.strip_prefix('#')?;
|
||||
let pointer = urlencoding::decode(fragment).ok()?;
|
||||
let pointer = jsonptr::Pointer::parse(pointer.as_ref()).ok()?;
|
||||
|
||||
let (table_token, pointer) = pointer.split_front()?;
|
||||
let table = table_token.decoded();
|
||||
let table = DEFINITION_TABLE_KEYS
|
||||
.into_iter()
|
||||
.find(|candidate| table.as_ref() == *candidate)?;
|
||||
|
||||
// Responses API non-strict mode accepts nested local refs such as
|
||||
// `#/$defs/User/properties/name`, so keep the parent definition reachable.
|
||||
let (name, _) = pointer.split_front()?;
|
||||
Some(DefinitionPointer {
|
||||
table,
|
||||
name: name.decoded().into_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
fn normalized_schema_types(
|
||||
map: &serde_json::Map<String, JsonValue>,
|
||||
) -> Vec<JsonSchemaPrimitiveType> {
|
||||
|
||||
@@ -848,3 +848,538 @@ fn parse_tool_input_schema_preserves_string_enum_constraints() {
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_refs_and_prunes_unreachable_defs() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": { "user": { "$ref": "#/$defs/User" } },
|
||||
// "$defs": {
|
||||
// "User": { "type": "object", "properties": { "name": { "type": "string" } } },
|
||||
// "Unused": { "type": "string" }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - Local `$ref` is preserved as a schema hint.
|
||||
// - Reachable `$defs` entries stay attached to the root schema.
|
||||
// - Unreachable `$defs` entries are pruned.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user": {"$ref": "#/$defs/User"}
|
||||
},
|
||||
"$defs": {
|
||||
"User": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"Unused": {"type": "string"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"user".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/User".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
defs: Some(BTreeMap::from([(
|
||||
"User".to_string(),
|
||||
JsonSchema::object(
|
||||
BTreeMap::from([(
|
||||
"name".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
)]),
|
||||
/*required*/ None,
|
||||
/*additional_properties*/ None,
|
||||
),
|
||||
)])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_refs_from_properties_named_def_tables() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": {
|
||||
// "$defs": { "$ref": "#/$defs/User" }
|
||||
// },
|
||||
// "$defs": { "User": { "type": "string" }, "Unused": { "type": "boolean" } }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - A property named like the `$defs` keyword is treated as a user field
|
||||
// while traversing `properties`.
|
||||
// - Refs from that property schema still mark root definitions reachable.
|
||||
// - Unreferenced root definitions are still pruned.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"$defs": {"$ref": "#/$defs/User"}
|
||||
},
|
||||
"$defs": {
|
||||
"User": {"type": "string"},
|
||||
"Unused": {"type": "boolean"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"$defs".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/User".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
defs: Some(BTreeMap::from([(
|
||||
"User".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
)])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_collects_refs_from_schema_child_keywords() {
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items_holder": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/$defs/Item"}
|
||||
},
|
||||
"map_holder": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"$ref": "#/$defs/Extra"}
|
||||
},
|
||||
"choice": {
|
||||
"anyOf": [
|
||||
{"$ref": "#/$defs/Choice"},
|
||||
{"type": "string"}
|
||||
]
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"Choice": {"type": "boolean"},
|
||||
"Extra": {"type": "number"},
|
||||
"Item": {"type": "string"},
|
||||
"Unused": {"type": "null"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(schema).expect("serialize schema"),
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"choice": {
|
||||
"anyOf": [
|
||||
{"$ref": "#/$defs/Choice"},
|
||||
{"type": "string"}
|
||||
]
|
||||
},
|
||||
"items_holder": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/$defs/Item"}
|
||||
},
|
||||
"map_holder": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": {"$ref": "#/$defs/Extra"}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"Choice": {"type": "boolean"},
|
||||
"Extra": {"type": "number"},
|
||||
"Item": {"type": "string"}
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_handles_cyclic_local_refs() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": { "node": { "$ref": "#/$defs/Node" } },
|
||||
// "$defs": {
|
||||
// "Node": {
|
||||
// "type": "object",
|
||||
// "properties": { "next": { "$ref": "#/$defs/Node" } }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - Recursive refs are preserved.
|
||||
// - Pruning traversal terminates after visiting each local target once.
|
||||
// - Responses API handles this recursive local-ref shape correctly.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"node": {"$ref": "#/$defs/Node"}
|
||||
},
|
||||
"$defs": {
|
||||
"Node": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"next": {"$ref": "#/$defs/Node"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"node".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/Node".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
defs: Some(BTreeMap::from([(
|
||||
"Node".to_string(),
|
||||
JsonSchema::object(
|
||||
BTreeMap::from([(
|
||||
"next".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/Node".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
/*required*/ None,
|
||||
/*additional_properties*/ None,
|
||||
),
|
||||
)])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_legacy_definitions() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": { "user": { "$ref": "#/definitions/User" } },
|
||||
// "definitions": {
|
||||
// "User": { "type": "object", "properties": { "profile": { "$ref": "#/definitions/Profile" } } },
|
||||
// "Profile": { "type": "object", "properties": { "name": { "type": "string" } } }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - Codex preserves legacy `definitions`.
|
||||
// - Reachability follows refs through the legacy definition table.
|
||||
// - Unreachable legacy definition entries are pruned.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user": {"$ref": "#/definitions/User"}
|
||||
},
|
||||
"definitions": {
|
||||
"User": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"profile": {"$ref": "#/definitions/Profile"}
|
||||
}
|
||||
},
|
||||
"Profile": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"Unused": {"type": "string"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"user".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/definitions/User".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
definitions: Some(BTreeMap::from([
|
||||
(
|
||||
"Profile".to_string(),
|
||||
JsonSchema::object(
|
||||
BTreeMap::from([(
|
||||
"name".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
)]),
|
||||
/*required*/ None,
|
||||
/*additional_properties*/ None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"User".to_string(),
|
||||
JsonSchema::object(
|
||||
BTreeMap::from([(
|
||||
"profile".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/definitions/Profile".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
/*required*/ None,
|
||||
/*additional_properties*/ None,
|
||||
),
|
||||
),
|
||||
])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_unresolved_and_external_refs() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": {
|
||||
// "missing": { "$ref": "#/$defs/Missing" },
|
||||
// "remote": { "$ref": "https://example.com/schema.json" }
|
||||
// },
|
||||
// "$defs": { "Unused": { "type": "string" } }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - Unresolved local refs and external refs are preserved.
|
||||
// - Unreachable local definitions are still pruned.
|
||||
// - Responses API handles these refs correctly during downstream validation.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"missing": {"$ref": "#/$defs/Missing"},
|
||||
"remote": {"$ref": "https://example.com/schema.json"}
|
||||
},
|
||||
"$defs": {
|
||||
"Unused": {"type": "string"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([
|
||||
(
|
||||
"missing".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/Missing".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"remote".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("https://example.com/schema.json".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_nested_defs_ref_parent() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": { "name": { "$ref": "#/$defs/User/properties/name" } },
|
||||
// "$defs": {
|
||||
// "User": { "type": "object", "properties": { "name": { "type": "string" } } },
|
||||
// "name": { "type": "string" },
|
||||
// "Unused": { "type": "boolean" }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - The nested JSON Pointer ref remains unchanged.
|
||||
// - The parent root definition is retained so the local ref does not dangle.
|
||||
// - Unreferenced root definitions are still pruned.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"$ref": "#/$defs/User/properties/name"}
|
||||
},
|
||||
"$defs": {
|
||||
"User": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"name": {"type": "string"},
|
||||
"Unused": {"type": "boolean"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"name".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/User/properties/name".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
defs: Some(BTreeMap::from([(
|
||||
"User".to_string(),
|
||||
JsonSchema::object(
|
||||
BTreeMap::from([(
|
||||
"name".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
)]),
|
||||
/*required*/ None,
|
||||
/*additional_properties*/ None,
|
||||
),
|
||||
)])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_preserves_percent_encoded_definition_refs() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": {
|
||||
// "user": { "$ref": "#/$defs/User%20Name" },
|
||||
// "profile": { "$ref": "#/%24defs/Profile%7E0Name" }
|
||||
// },
|
||||
// "$defs": {
|
||||
// "User Name": { "type": "string" },
|
||||
// "Profile~Name": { "type": "string" },
|
||||
// "Unused": { "type": "boolean" }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - URI fragment percent encoding is decoded before JSON Pointer `~`
|
||||
// escaping, per RFC 6901 section 6.
|
||||
// - The original `$ref` strings are preserved, but their definition
|
||||
// targets are recognized as reachable and retained.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user": {"$ref": "#/$defs/User%20Name"},
|
||||
"profile": {"$ref": "#/%24defs/Profile%7E0Name"}
|
||||
},
|
||||
"$defs": {
|
||||
"User Name": {"type": "string"},
|
||||
"Profile~Name": {"type": "string"},
|
||||
"Unused": {"type": "boolean"}
|
||||
}
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([
|
||||
(
|
||||
"profile".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/%24defs/Profile%7E0Name".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"user".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/User%20Name".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
])),
|
||||
defs: Some(BTreeMap::from([
|
||||
(
|
||||
"Profile~Name".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
),
|
||||
(
|
||||
"User Name".to_string(),
|
||||
JsonSchema::string(/*description*/ None),
|
||||
),
|
||||
])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_tool_input_schema_drops_malformed_definition_tables() {
|
||||
// Example schema shape:
|
||||
// {
|
||||
// "type": "object",
|
||||
// "properties": { "user": { "$ref": "#/$defs/User" } },
|
||||
// "$defs": ["not", "an", "object"]
|
||||
// }
|
||||
//
|
||||
// Expected normalization behavior:
|
||||
// - Malformed `$defs` tables are dropped instead of rejecting the schema.
|
||||
// - The unresolved local ref remains visible to the model.
|
||||
let schema = parse_tool_input_schema(&serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user": {"$ref": "#/$defs/User"}
|
||||
},
|
||||
"$defs": ["not", "an", "object"]
|
||||
}))
|
||||
.expect("parse schema");
|
||||
|
||||
assert_eq!(
|
||||
schema,
|
||||
JsonSchema {
|
||||
schema_type: Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Object)),
|
||||
properties: Some(BTreeMap::from([(
|
||||
"user".to_string(),
|
||||
JsonSchema {
|
||||
schema_ref: Some("#/$defs/User".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)])),
|
||||
..Default::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user