AWS SDK

AWS SDK

rev. 0e89e4bb6e5f1c92f5acf1c6677e026389dfc4be

Files changed:

tmp-codegen-diff/aws-sdk/sdk/aws-runtime/src/static_stability.rs

@@ -21,21 +80,83 @@
   41     41   
use tracing::Instrument;
   42     42   
   43     43   
// Blocking refresh point before expiration
   44     44   
const DEFAULT_MANDATORY_WINDOW: Duration = Duration::from_secs(60);
   45     45   
// Refresh point before expiry for caching-only (ineligible) identities
   46     46   
const CACHING_ONLY_BUFFER_TIME: Duration = Duration::from_secs(10);
   47     47   
// Uniform backoff floor after a failed refresh.
   48     48   
const BACKOFF_MIN_SECS: u64 = 300;
   49     49   
// Uniform backoff jitter span (300..=600s total).
   50     50   
const BACKOFF_JITTER_SECS: u64 = 300;
          51  +
// Non-recoverable error cache floor + jitter span (1..=5s total).
          52  +
const ERROR_CACHE_MIN_SECS: u64 = 1;
          53  +
const ERROR_CACHE_JITTER_SECS: u64 = 4;
   51     54   
   52     55   
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_millis(3100);
   53     56   
   54     57   
// NOTE: `pessimistic_load_timeout` below is deliberately duplicated from
   55     58   
// `aws_smithy_runtime::client::identity::cache::lazy::pessimistic_load_timeout` rather than
   56     59   
// imported, to avoid the one-way `pub` API exposure.
   57     60   
//
   58     61   
// Derive a pessimistic load timeout from the configured retry/timeout so the source's own retries
   59     62   
// can finish before the cache kills the future.
   60     63   
fn pessimistic_load_timeout(config_bag: &ConfigBag) -> Duration {
@@ -154,157 +213,228 @@
  174    177   
    // Refresh a partition from the source, then commit (success) or serve-cached/back-off/raise
  175    178   
    // (failure). The source `.await` is held under the async `refresh_gate` only (acquired by the
  176    179   
    // caller); the sync `state` lock is taken only for the brief snapshot/commit sections.
  177    180   
    async fn refresh(
  178    181   
        &self,
  179    182   
        part: &Partition,
  180    183   
        resolver: &SharedIdentityResolver,
  181    184   
        runtime_components: &RuntimeComponents,
  182    185   
        config_bag: &ConfigBag,
  183    186   
    ) -> Result<Identity, BoxError> {
         187  +
        // Non-recoverable error cache: if a terminal failure is still cached, re-raise it without
         188  +
        // contacting the source, so a caller looping on the error cannot hammer the source. Checked
         189  +
        // here — the caller already holds the refresh gate — before any source call.
         190  +
        {
         191  +
            let st = part.state.lock().unwrap();
         192  +
            if let (Some(err), Some(expires_at)) = (&st.cached_error, st.cached_error_expires_at) {
         193  +
                if runtime_components.time_source().expect("validated").now() < expires_at {
         194  +
                    return Err(err.clone().into());
         195  +
                }
         196  +
            }
         197  +
        }
         198  +
  184    199   
        let prev = part.state.lock().unwrap().cached.clone();
  185    200   
  186    201   
        let sleep_impl = runtime_components.sleep_impl().expect("validated");
  187    202   
        let load_timeout = self
  188    203   
            .load_timeout
  189    204   
            .unwrap_or_else(|| pessimistic_load_timeout(config_bag));
  190    205   
        let timeout_future = sleep_impl.sleep(load_timeout);
  191    206   
        let resolved: Result<Identity, BoxError> = async move {
  192    207   
            match Timeout::new(
  193    208   
                resolver.resolve_identity(runtime_components, config_bag),
@@ -223,238 +289,314 @@
  243    258   
                    // Non-expiring identity: no expiration-based refresh. Served indefinitely
  244    259   
                    // (classify returns Valid when `expiration` is None); only invalidation, which
  245    260   
                    // sets `expiration = now`, forces a refresh.
  246    261   
                    None => {
  247    262   
                        st.expiration = None;
  248    263   
                        st.advisory_at = None;
  249    264   
                        st.mandatory_at = None;
  250    265   
                    }
  251    266   
                }
  252    267   
                st.next_refresh_allowed_at = None; // clear backoff
         268  +
                st.cached_error = None; // a success clears any cached terminal error
         269  +
                st.cached_error_expires_at = None;
  253    270   
                Ok(id)
  254    271   
            }
  255    272   
            Err(err) => {
  256         -
                // A non-recoverable error raises immediately — before backoff and
  257         -
                // before serve-cached. The cache holds only a predicate, naming no error types.
         273  +
                // A non-recoverable error raises immediately — before backoff and before
         274  +
                // serve-cached. The cache holds only a predicate, naming no error types.
  258    275   
                if self.non_recoverable.as_ref().is_some_and(|p| p(&err)) {
  259         -
                    return Err(err);
         276  +
                    // Cache the terminal error for a short, jittered window so repeat callers are
         277  +
                    // served it without another source call; still raise it to this caller now.
         278  +
                    let cached = CachedNonRecoverableError(Arc::from(err));
         279  +
                    {
         280  +
                        let mut st = part.state.lock().unwrap();
         281  +
                        st.cached_error = Some(cached.clone());
         282  +
                        st.cached_error_expires_at = Some(now + jittered_error_cache());
         283  +
                    }
         284  +
                    return Err(cached.into());
  260    285   
                }
  261    286   
                let mut st = part.state.lock().unwrap();
  262    287   
                // Backoff AND serve-stale are BOTH static stability — gated on eligibility.
  263    288   
                match prev {
  264    289   
                    Some(c) if eligible(&c) => {
  265    290   
                        st.next_refresh_allowed_at =
  266    291   
                            Some(now + self.backoff_override.unwrap_or_else(jittered_backoff));
  267    292   
                        tracing::warn!(
  268    293   
                            error = ?err,
  269    294   
                            "credential refresh failed; serving cached credentials (static stability)"
@@ -362,387 +421,449 @@
  382    407   
struct CachedState {
  383    408   
    // Retain-always; only ever *replaced* on success, never cleared.
  384    409   
    cached: Option<Identity>,
  385    410   
    expiration: Option<SystemTime>,
  386    411   
    // Non-blocking refresh point.
  387    412   
    advisory_at: Option<SystemTime>,
  388    413   
    // Blocking refresh point (<= expiration).
  389    414   
    mandatory_at: Option<SystemTime>,
  390    415   
    // Backoff gate after a failed refresh.
  391    416   
    next_refresh_allowed_at: Option<SystemTime>,
         417  +
    // Short-lived, jittered cache of a non-recoverable error, re-raised to repeat callers.
         418  +
    cached_error: Option<CachedNonRecoverableError>,
         419  +
    cached_error_expires_at: Option<SystemTime>,
  392    420   
}
  393    421   
  394    422   
enum Decision {
  395    423   
    Valid(Identity),
  396    424   
    RateLimited(Identity),
  397    425   
    Advisory(Identity),
  398    426   
    Mandatory,
  399    427   
    Initial,
  400    428   
}
  401    429   
@@ -427,455 +486,536 @@
  447    475   
        Duration::from_secs(15 * 60)
  448    476   
    } else {
  449    477   
        Duration::from_secs(60 * 60)
  450    478   
    }
  451    479   
}
  452    480   
  453    481   
fn jittered_backoff() -> Duration {
  454    482   
    Duration::from_secs(BACKOFF_MIN_SECS + fastrand::u64(0..=BACKOFF_JITTER_SECS))
  455    483   
}
  456    484   
         485  +
fn jittered_error_cache() -> Duration {
         486  +
    Duration::from_secs(ERROR_CACHE_MIN_SECS + fastrand::u64(0..=ERROR_CACHE_JITTER_SECS))
         487  +
}
         488  +
         489  +
// A `Clone` wrapper so a cached non-recoverable error can be re-raised to multiple callers
         490  +
// (`BoxError` is not `Clone`). The underlying error is exposed through `source`, matching how
         491  +
// `Arc`-wrapped errors elsewhere in the runtime surface their cause.
         492  +
#[derive(Clone, Debug)]
         493  +
struct CachedNonRecoverableError(Arc<dyn Error + Send + Sync>);
         494  +
         495  +
impl fmt::Display for CachedNonRecoverableError {
         496  +
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         497  +
        fmt::Display::fmt(&self.0, f)
         498  +
    }
         499  +
}
         500  +
         501  +
impl Error for CachedNonRecoverableError {
         502  +
    fn source(&self) -> Option<&(dyn Error + 'static)> {
         503  +
        Some(self.0.as_ref())
         504  +
    }
         505  +
}
         506  +
  457    507   
// Default non-recoverable predicate injected into the cache by the AWS layer: a terminal
  458    508   
// `CredentialsError::Unrecoverable` anywhere in the source chain bypasses backoff and static
  459    509   
// stability. Providers such as `ChainProvider` wrap the base-provider error, so walk the chain
  460    510   
// rather than inspecting only the outermost error.
  461    511   
fn aws_non_recoverable(err: &BoxError) -> bool {
  462    512   
    let mut source: Option<&(dyn Error + 'static)> = Some(&**err);
  463    513   
    while let Some(e) = source {
  464    514   
        if e.downcast_ref::<CredentialsError>()
  465    515   
            .is_some_and(CredentialsError::is_unrecoverable)
  466    516   
        {
@@ -690,740 +773,821 @@
  710    760   
                            let got = r.expect("newCredentials");
  711    761   
                            cached_id = fresh_ids.pop_front().expect("a queued fresh id");
  712    762   
                            assert_eq!(id_of(&got), cached_id, "{doc}: new id");
  713    763   
                            served = Some(got);
  714    764   
                        }
  715    765   
                        "noCredentialsError" => assert!(r.is_err(), "{doc}: noCredentialsError"),
  716    766   
                        "nonRecoverableError" => {
  717    767   
                            let Err(e) = r else {
  718    768   
                                panic!("{doc}: expected nonRecoverableError");
  719    769   
                            };
  720         -
                            assert!(
  721         -
                                e.downcast_ref::<CredentialsError>()
  722         -
                                    .is_some_and(CredentialsError::is_unrecoverable),
  723         -
                                "{doc}: nonRecoverableError"
  724         -
                            );
         770  +
                            // The error may be the cache's `CachedNonRecoverableError` wrapper, so
         771  +
                            // check the source chain (the same detection the cache uses).
         772  +
                            assert!(aws_non_recoverable(&e), "{doc}: nonRecoverableError");
  725    773   
                        }
  726    774   
                        other => panic!("{doc}: unknown result {other:?}"),
  727    775   
                    }
  728    776   
                    if let Some(w) = expected.advisory_window_seconds {
  729    777   
                        assert_eq!(
  730    778   
                            cache.advisory_window(),
  731    779   
                            Some(Duration::from_secs(w)),
  732    780   
                            "{doc}: advisoryWindowSeconds"
  733    781   
                        );
  734    782   
                    }
  735    783   
                }
  736    784   
            }
  737    785   
        }
  738    786   
    }
  739    787   
  740    788   
    #[tokio::test]
  741    789   
    async fn test_suite() {
  742    790   
        let scenarios: Vec<Scenario> = serde_json::from_str(SUITE).expect("valid suite json");
  743         -
        assert_eq!(scenarios.len(), 23, "expected the full modeled suite");
         791  +
        assert_eq!(scenarios.len(), 24, "expected the full modeled suite");
  744    792   
        for s in &scenarios {
  745    793   
            run_scenario(s).await;
  746    794   
        }
  747    795   
    }
  748    796   
  749    797   
    #[derive(Debug, Clone, PartialEq)]
  750    798   
    struct TestCreds {
  751    799   
        id: u32,
  752    800   
    }
  753    801   
@@ -834,882 +893,954 @@
  854    902   
        let m = |mins: u64| Duration::from_secs(mins * 60);
  855    903   
        assert_eq!(advisory_window_for(m(10)), m(5));
  856    904   
        assert_eq!(advisory_window_for(m(20)), m(5)); // boundary, inclusive
  857    905   
        assert_eq!(advisory_window_for(m(21)), m(15));
  858    906   
        assert_eq!(advisory_window_for(m(60)), m(15));
  859    907   
        assert_eq!(advisory_window_for(m(89)), m(15));
  860    908   
        assert_eq!(advisory_window_for(m(90)), m(60)); // boundary
  861    909   
        assert_eq!(advisory_window_for(m(6 * 60)), m(60));
  862    910   
    }
  863    911   
         912  +
    // The JSON suite's determinism relies on this window staying strictly within (0s, 6s): an
         913  +
    // immediate retry (0s elapsed) must still be cached, and a 6s advance must always clear it.
         914  +
    #[test]
         915  +
    fn jittered_error_cache_bounds() {
         916  +
        for _ in 0..1000 {
         917  +
            let d = jittered_error_cache();
         918  +
            assert!(
         919  +
                (Duration::from_secs(1)..=Duration::from_secs(5)).contains(&d),
         920  +
                "error-cache window {d:?} out of 1..=5s"
         921  +
            );
         922  +
        }
         923  +
    }
         924  +
  864    925   
    // A non-recoverable error wrapped by an outer provider error (as `ChainProvider` produces) is
  865    926   
    // still detected by walking the source chain, not just the outermost error.
  866    927   
    #[test]
  867    928   
    fn non_recoverable_walks_source_chain() {
  868    929   
        let wrapped: BoxError =
  869    930   
            CredentialsError::provider_error(CredentialsError::unrecoverable("expired SSO token"))
  870    931   
                .into();
  871    932   
        assert!(
  872    933   
            aws_non_recoverable(&wrapped),
  873    934   
            "a nested Unrecoverable must be detected"
@@ -1151,1212 +1181,1304 @@
 1171   1232   
        // Every caller receives the one refreshed identity; only one refresh contacted the source.
 1172   1233   
        assert_eq!(id_of(&r1.unwrap()), 2);
 1173   1234   
        assert_eq!(id_of(&r2.unwrap()), 2);
 1174   1235   
        assert_eq!(id_of(&r3.unwrap()), 2);
 1175   1236   
        assert_eq!(
 1176   1237   
            contacts.load(Ordering::SeqCst),
 1177   1238   
            2,
 1178   1239   
            "one refresh shared by all waiters"
 1179   1240   
        );
 1180   1241   
    }
        1242  +
        1243  +
    // Concurrency + non-recoverable: the one caller that refreshes fails non-recoverably and
        1244  +
    // caches the error; the waiters reuse the cached error instead of each re-contacting the
        1245  +
    // source.
        1246  +
    #[tokio::test]
        1247  +
    async fn mandatory_concurrent_non_recoverable_reuses_cached_error() {
        1248  +
        let time = ManualTimeSource::new(epoch(0));
        1249  +
        let (_tick, sleep) = tick_advance_time_and_sleep();
        1250  +
        let components = RuntimeComponentsBuilder::for_tests()
        1251  +
            .with_time_source(Some(time.clone()))
        1252  +
            .with_sleep_impl(Some(sleep))
        1253  +
            .build()
        1254  +
            .unwrap();
        1255  +
        let cfg = ConfigBag::base();
        1256  +
        let gate = Arc::new(tokio::sync::Notify::new());
        1257  +
        let contacts = Arc::new(AtomicUsize::new(0));
        1258  +
        let resolver = SharedIdentityResolver::new(GatedSource {
        1259  +
            gate: gate.clone(),
        1260  +
            contacts: contacts.clone(),
        1261  +
            results: Mutex::new(vec![
        1262  +
                Ok(identity(1, 3600, true)),
        1263  +
                Err(CredentialsError::unrecoverable("terminal").into()),
        1264  +
            ]),
        1265  +
        });
        1266  +
        let cache = StaticStabilityCache::builder().build();
        1267  +
        1268  +
        gate.notify_one();
        1269  +
        let seed = cache
        1270  +
            .resolve_cached_identity(resolver.clone(), &components, &cfg)
        1271  +
            .await
        1272  +
            .unwrap();
        1273  +
        assert_eq!(id_of(&seed), 1);
        1274  +
        assert_eq!(contacts.load(Ordering::SeqCst), 1);
        1275  +
        1276  +
        time.set_time(epoch(3700)); // expired -> mandatory path
        1277  +
        1278  +
        // Three concurrent callers + a driver that releases the single in-flight refresh.
        1279  +
        let get1 = cache.resolve_cached_identity(resolver.clone(), &components, &cfg);
        1280  +
        let get2 = cache.resolve_cached_identity(resolver.clone(), &components, &cfg);
        1281  +
        let get3 = cache.resolve_cached_identity(resolver.clone(), &components, &cfg);
        1282  +
        let driver = async {
        1283  +
            tokio::task::yield_now().await;
        1284  +
            gate.notify_one();
        1285  +
        };
        1286  +
        let (r1, r2, r3, ()) = tokio::join!(get1, get2, get3, driver);
        1287  +
        1288  +
        // Every caller receives the non-recoverable error; only one contacted the source.
        1289  +
        for r in [r1, r2, r3] {
        1290  +
            let Err(e) = r else {
        1291  +
                panic!("expected a non-recoverable error");
        1292  +
            };
        1293  +
            assert!(
        1294  +
                aws_non_recoverable(&e),
        1295  +
                "each waiter gets the non-recoverable error"
        1296  +
            );
        1297  +
        }
        1298  +
        assert_eq!(
        1299  +
            contacts.load(Ordering::SeqCst),
        1300  +
            2,
        1301  +
            "one refresh failed for all callers; the cache suppressed re-contacts"
        1302  +
        );
        1303  +
    }
 1181   1304   
}

tmp-codegen-diff/aws-sdk/sdk/aws-runtime/test-data/credential-refresh-tests.json

@@ -161,161 +255,280 @@
  181    181   
      {
  182    182   
        "type": "getCredentials",
  183    183   
        "response": "freshCredentials",
  184    184   
        "lifetimeSeconds": 21600,
  185    185   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false, "advisoryWindowSeconds": 1800 }
  186    186   
      }
  187    187   
    ]
  188    188   
  },
  189    189   
  190    190   
  {
  191         -
    "documentation": "Advisory window, non-recoverable failure: the SDK raises immediately. Because no refresh backoff is applied, the next call contacts the source again rather than being rate limited.",
         191  +
    "documentation": "Advisory window, non-recoverable failure: the SDK raises immediately. No refresh backoff is applied, but the error is cached for up to 5 seconds, so a recovering call succeeds once that cache expires.",
  192    192   
    "given": { "cachedCredentials": "advisory" },
  193    193   
    "steps": [
  194    194   
      {
  195    195   
        "type": "getCredentials",
  196    196   
        "response": "nonRecoverableError",
  197    197   
        "documentation": "Non-recoverable failure: the SDK raises and does not apply the refresh backoff.",
  198    198   
        "expected": { "result": "nonRecoverableError", "sourceContacted": true, "rateLimited": false }
  199    199   
      },
         200  +
      {
         201  +
        "type": "advanceTime",
         202  +
        "seconds": 6
         203  +
      },
  200    204   
      {
  201    205   
        "type": "getCredentials",
  202    206   
        "response": "freshCredentials",
  203         -
        "documentation": "No refresh backoff was applied, so this call contacts the source again and succeeds.",
         207  +
        "documentation": "The non-recoverable error cache (max 5 seconds) has expired, so this call contacts the source again and succeeds.",
  204    208   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false }
  205    209   
      }
  206    210   
    ]
  207    211   
  },
  208    212   
  {
  209         -
    "documentation": "Mandatory window, non-recoverable failure: the SDK raises immediately. Because no refresh backoff is applied, the next call contacts the source again rather than being rate limited.",
         213  +
    "documentation": "Mandatory window, non-recoverable failure: the SDK raises immediately. No refresh backoff is applied, but the error is cached for up to 5 seconds, so a recovering call succeeds once that cache expires.",
  210    214   
    "given": { "cachedCredentials": "mandatory" },
  211    215   
    "steps": [
  212    216   
      {
  213    217   
        "type": "getCredentials",
  214    218   
        "response": "nonRecoverableError",
  215    219   
        "documentation": "Non-recoverable failure: the SDK raises and does not apply the refresh backoff.",
  216    220   
        "expected": { "result": "nonRecoverableError", "sourceContacted": true, "rateLimited": false }
  217    221   
      },
         222  +
      {
         223  +
        "type": "advanceTime",
         224  +
        "seconds": 6
         225  +
      },
  218    226   
      {
  219    227   
        "type": "getCredentials",
  220    228   
        "response": "freshCredentials",
  221         -
        "documentation": "No refresh backoff was applied, so this call contacts the source again and succeeds.",
         229  +
        "documentation": "The non-recoverable error cache (max 5 seconds) has expired, so this call contacts the source again and succeeds.",
  222    230   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false }
  223    231   
      }
  224    232   
    ]
  225    233   
  },
         234  +
  {
         235  +
    "documentation": "Non-recoverable error, then an immediate retry with no clock advance: the error is still cached, so the SDK re-raises it without contacting the source. This protects the credential source from an application that swallows the error and retries in a loop.",
         236  +
    "given": { "cachedCredentials": "advisory" },
         237  +
    "steps": [
         238  +
      {
         239  +
        "type": "getCredentials",
         240  +
        "response": "nonRecoverableError",
         241  +
        "documentation": "Non-recoverable failure: the SDK raises and caches the error for up to 5 seconds.",
         242  +
        "expected": { "result": "nonRecoverableError", "sourceContacted": true, "rateLimited": false }
         243  +
      },
         244  +
      {
         245  +
        "type": "getCredentials",
         246  +
        "documentation": "Immediate retry with no clock advance. The cached error is still active, so the SDK re-raises it without contacting the source.",
         247  +
        "expected": { "result": "nonRecoverableError", "sourceContacted": false, "rateLimited": false }
         248  +
      }
         249  +
    ]
         250  +
  },
  226    251   
  227    252   
  {
  228    253   
    "documentation": "Invalidate with an access key ID matching the cached credentials routes the next getCredentials through the mandatory refresh path, and the refresh succeeds.",
  229    254   
    "given": { "cachedCredentials": "valid", "accessKeyId": "AKID-1" },
  230    255   
    "steps": [
  231    256   
      { "type": "invalidate", "rejectedAccessKeyId": "AKID-1" },
  232    257   
      {
  233    258   
        "type": "getCredentials",
  234    259   
        "response": "freshCredentials",
  235    260   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false }
@@ -297,322 +344,373 @@
  317    342   
      },
  318    343   
      {
  319    344   
        "type": "getCredentials",
  320    345   
        "response": "freshCredentials",
  321    346   
        "documentation": "725s elapsed total and the refresh backoff has elapsed, so the SDK contacts the credential source and the refresh succeeds.",
  322    347   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false }
  323    348   
      }
  324    349   
    ]
  325    350   
  },
  326    351   
  {
  327         -
    "documentation": "No cached credentials and the initial fetch fails with a non-recoverable error: the SDK raises the error directly rather than a generic NoCredentialsError. Because no refresh backoff is applied, the next call contacts the source again.",
         352  +
    "documentation": "No cached credentials and the initial fetch fails with a non-recoverable error: the SDK raises the error directly rather than a generic NoCredentialsError. No refresh backoff is applied, but the error is cached for up to 5 seconds, so a recovering call succeeds once that cache expires.",
  328    353   
    "given": { "cachedCredentials": "none" },
  329    354   
    "steps": [
  330    355   
      {
  331    356   
        "type": "getCredentials",
  332    357   
        "response": "nonRecoverableError",
  333    358   
        "documentation": "Non-recoverable failure: the SDK raises and does not apply the refresh backoff.",
  334    359   
        "expected": { "result": "nonRecoverableError", "sourceContacted": true, "rateLimited": false }
  335    360   
      },
         361  +
      {
         362  +
        "type": "advanceTime",
         363  +
        "seconds": 6
         364  +
      },
  336    365   
      {
  337    366   
        "type": "getCredentials",
  338    367   
        "response": "freshCredentials",
  339         -
        "documentation": "No refresh backoff was applied, so this call contacts the source again and succeeds.",
         368  +
        "documentation": "The non-recoverable error cache (max 5 seconds) has expired, so this call contacts the source again and succeeds.",
  340    369   
        "expected": { "result": "newCredentials", "sourceContacted": true, "rateLimited": false }
  341    370   
      }
  342    371   
    ]
  343    372   
  }
  344    373   
]