chore: stop app-server auth refresh storms after permanent token failure (#15530)

built from #14256. PR description from @etraut-openai:

This PR addresses a hole in [PR
11802](https://github.com/openai/codex/pull/11802). The previous PR
assumed that app server clients would respond to token refresh failures
by presenting the user with an error ("you must log in again") and then
not making further attempts to call network endpoints using the expired
token. While they do present the user with this error, they don't
prevent further attempts to call network endpoints and can repeatedly
call `getAuthStatus(refreshToken=true)` resulting in many failed calls
to the token refresh endpoint.

There are three solutions I considered here:
1. Change the getAuthStatus app server call to return a null auth if the
caller specified "refreshToken" on input and the refresh attempt fails.
This will cause clients to immediately log out the user and return them
to the log in screen. This is a really bad user experience. It's also a
breaking change in the app server contract that could break third-party
clients.
2. Augment the getAuthStatus app server call to return an additional
field that indicates the state of "token could not be refreshed". This
is a non-breaking change to the app server API, but it requires
non-trivial changes for all clients to properly handle this new field
properly.
3. Change the getAuthStatus implementation to handle the case where a
token refresh fails by marking the AuthManager's in-memory access and
refresh tokens as "poisoned" so it they are no longer used. This is the
simplest fix that requires no client changes.

I chose option 3.

Here's Codex's explanation of this change:

When an app-server client asks `getAuthStatus(refreshToken=true)`, we
may try to refresh a stale ChatGPT access token. If that refresh fails
permanently (for example `refresh_token_reused`, expired, or revoked),
the old behavior was bad in two ways:

1. We kept the in-memory auth snapshot alive as if it were still usable.
2. Later auth checks could retry refresh again and again, creating a
storm of doomed `/oauth/token` requests and repeatedly surfacing the
same failure.

This is especially painful for app-server clients because they poll auth
status and can keep driving the refresh path without any real chance of
recovery.

This change makes permanent refresh failures terminal for the current
managed auth snapshot without changing the app-server API contract.

What changed:
- `AuthManager` now poisons the current managed auth snapshot in memory
after a permanent refresh failure, keyed to the unchanged `AuthDotJson`.
- Once poisoned, later refresh attempts for that same snapshot fail fast
locally without calling the auth service again.
- The poison is cleared automatically when auth materially changes, such
as a new login, logout, or reload of different auth state from storage.
- `getAuthStatus(includeToken=true)` now omits `authToken` after a
permanent refresh failure instead of handing out the stale cached bearer
token.

This keeps the current auth method visible to clients, avoids forcing an
immediate logout flow, and stops repeated refresh attempts for
credentials that cannot recover.

---------

Co-authored-by: Eric Traut <etraut@openai.com>

codex-rs/login/src/auth/auth_tests.rs

@@ -197,6 +197,49 @@ fn unauthorized_recovery_reports_mode_and_step_names() {
     assert_eq!(external.step_name(), "external_refresh");
 }
 
+#[test]
+fn refresh_failure_is_scoped_to_the_matching_auth_snapshot() {
+    let codex_home = tempdir().unwrap();
+    write_auth_file(
+        AuthFileParams {
+            openai_api_key: None,
+            chatgpt_plan_type: Some("pro".to_string()),
+            chatgpt_account_id: Some("org_mine".to_string()),
+        },
+        codex_home.path(),
+    )
+    .expect("failed to write auth file");
+
+    let auth = super::load_auth(codex_home.path(), false, AuthCredentialsStoreMode::File)
+        .expect("load auth")
+        .expect("auth available");
+    let mut updated_auth_dot_json = auth
+        .get_current_auth_json()
+        .expect("AuthDotJson should exist");
+    let updated_tokens = updated_auth_dot_json
+        .tokens
+        .as_mut()
+        .expect("tokens should exist");
+    updated_tokens.access_token = "new-access-token".to_string();
+    updated_tokens.refresh_token = "new-refresh-token".to_string();
+    let updated_auth = CodexAuth::from_auth_dot_json(
+        codex_home.path(),
+        updated_auth_dot_json,
+        AuthCredentialsStoreMode::File,
+    )
+    .expect("updated auth should parse");
+
+    let manager = AuthManager::from_auth_for_testing(auth.clone());
+    let error = RefreshTokenFailedError::new(
+        RefreshTokenFailedReason::Exhausted,
+        "refresh token already used",
+    );
+    manager.record_permanent_refresh_failure_if_unchanged(&auth, &error);
+
+    assert_eq!(manager.refresh_failure_for_auth(&auth), Some(error));
+    assert_eq!(manager.refresh_failure_for_auth(&updated_auth), None);
+}
+
 struct AuthFileParams {
     openai_api_key: Option<String>,
     chatgpt_plan_type: Option<String>,

codex-rs/login/src/auth/manager.rs

@@ -796,6 +796,15 @@ struct CachedAuth {
     auth: Option<CodexAuth>,
     /// Callback used to refresh external auth by asking the parent app for new tokens.
     external_refresher: Option<Arc<dyn ExternalAuthRefresher>>,
+    /// Permanent refresh failure cached for the current auth snapshot so
+    /// later refresh attempts for the same credentials fail fast without network.
+    permanent_refresh_failure: Option<AuthScopedRefreshFailure>,
+}
+
+#[derive(Clone)]
+struct AuthScopedRefreshFailure {
+    auth: CodexAuth,
+    error: RefreshTokenFailedError,
 }
 
 impl Debug for CachedAuth {
@@ -809,6 +818,13 @@ impl Debug for CachedAuth {
                 "external_refresher",
                 &self.external_refresher.as_ref().map(|_| "present"),
             )
+            .field(
+                "permanent_refresh_failure",
+                &self
+                    .permanent_refresh_failure
+                    .as_ref()
+                    .map(|failure| failure.error.reason),
+            )
             .finish()
     }
 }
@@ -1046,6 +1062,7 @@ impl AuthManager {
             inner: RwLock::new(CachedAuth {
                 auth: managed_auth,
                 external_refresher: None,
+                permanent_refresh_failure: None,
             }),
             enable_codex_api_key_env,
             auth_credentials_store_mode,
@@ -1058,6 +1075,7 @@ impl AuthManager {
         let cached = CachedAuth {
             auth: Some(auth),
             external_refresher: None,
+            permanent_refresh_failure: None,
         };
 
         Arc::new(Self {
@@ -1074,6 +1092,7 @@ impl AuthManager {
         let cached = CachedAuth {
             auth: Some(auth),
             external_refresher: None,
+            permanent_refresh_failure: None,
         };
         Arc::new(Self {
             codex_home,
@@ -1089,6 +1108,16 @@ impl AuthManager {
         self.inner.read().ok().and_then(|c| c.auth.clone())
     }
 
+    pub fn refresh_failure_for_auth(&self, auth: &CodexAuth) -> Option<RefreshTokenFailedError> {
+        self.inner.read().ok().and_then(|cached| {
+            cached
+                .permanent_refresh_failure
+                .as_ref()
+                .filter(|failure| Self::auths_equal_for_refresh(Some(auth), Some(&failure.auth)))
+                .map(|failure| failure.error.clone())
+        })
+    }
+
     /// Current cached auth (clone). May be `None` if not logged in or load failed.
     /// For stale managed ChatGPT auth, first performs a guarded reload and then
     /// refreshes only if the on-disk auth is unchanged.
@@ -1166,6 +1195,25 @@ impl AuthManager {
         }
     }
 
+    /// Records a permanent refresh failure only if the failed refresh was
+    /// attempted against the auth snapshot that is still cached.
+    fn record_permanent_refresh_failure_if_unchanged(
+        &self,
+        attempted_auth: &CodexAuth,
+        error: &RefreshTokenFailedError,
+    ) {
+        if let Ok(mut guard) = self.inner.write() {
+            let current_auth_matches =
+                Self::auths_equal_for_refresh(Some(attempted_auth), guard.auth.as_ref());
+            if current_auth_matches {
+                guard.permanent_refresh_failure = Some(AuthScopedRefreshFailure {
+                    auth: attempted_auth.clone(),
+                    error: error.clone(),
+                });
+            }
+        }
+    }
+
     fn load_auth_from_storage(&self) -> Option<CodexAuth> {
         load_auth(
             &self.codex_home,
@@ -1180,6 +1228,11 @@ impl AuthManager {
         if let Ok(mut guard) = self.inner.write() {
             let previous = guard.auth.as_ref();
             let changed = !AuthManager::auths_equal(previous, new_auth.as_ref());
+            let auth_changed_for_refresh =
+                !Self::auths_equal_for_refresh(previous, new_auth.as_ref());
+            if auth_changed_for_refresh {
+                guard.permanent_refresh_failure = None;
+            }
             tracing::info!("Reloaded auth, changed: {changed}");
             guard.auth = new_auth;
             changed
@@ -1255,6 +1308,12 @@ impl AuthManager {
     /// token is the same as the cached, then ask the token authority to refresh.
     pub async fn refresh_token(&self) -> Result<(), RefreshTokenError> {
         let auth_before_reload = self.auth_cached();
+        if auth_before_reload
+            .as_ref()
+            .is_some_and(CodexAuth::is_api_key_auth)
+        {
+            return Ok(());
+        }
         let expected_account_id = auth_before_reload
             .as_ref()
             .and_then(CodexAuth::get_account_id);
@@ -1285,7 +1344,12 @@ impl AuthManager {
             Some(auth) => auth,
             None => return Ok(()),
         };
-        match auth {
+        if let Some(error) = self.refresh_failure_for_auth(&auth) {
+            return Err(RefreshTokenError::Permanent(error));
+        }
+
+        let attempted_auth = auth.clone();
+        let result = match auth {
             CodexAuth::ChatgptAuthTokens(_) => {
                 self.refresh_external_auth(ExternalAuthRefreshReason::Unauthorized)
                     .await
@@ -1297,11 +1361,14 @@ impl AuthManager {
                     ))
                 })?;
                 self.refresh_and_persist_chatgpt_token(&chatgpt_auth, token_data.refresh_token)
-                    .await?;
-                Ok(())
+                    .await
             }
             CodexAuth::ApiKey(_) => Ok(()),
+        };
+        if let Err(RefreshTokenError::Permanent(error)) = &result {
+            self.record_permanent_refresh_failure_if_unchanged(&attempted_auth, error);
         }
+        result
     }
 
     /// Log out by deleting the on‑disk auth.json (if present). Returns Ok(true)