aws_smithy_runtime/client/retries/strategy/
standard.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use std::sync::{Mutex, OnceLock};
7use std::time::{Duration, SystemTime};
8
9use tokio::sync::OwnedSemaphorePermit;
10use tracing::{debug, trace};
11
12use aws_smithy_runtime_api::box_error::BoxError;
13use aws_smithy_runtime_api::client::interceptors::context::{
14    BeforeTransmitInterceptorContextMut, InterceptorContext,
15};
16use aws_smithy_runtime_api::client::interceptors::{dyn_dispatch_hint, Intercept};
17use aws_smithy_runtime_api::client::retries::classifiers::{RetryAction, RetryReason};
18use aws_smithy_runtime_api::client::retries::{RequestAttempts, RetryStrategy, ShouldAttempt};
19use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
20use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace};
21use aws_smithy_types::retry::{ErrorKind, RetryConfig, RetryMode, RetrySpec};
22
23use crate::client::retries::classifiers::run_classifiers_on_ctx;
24use crate::client::retries::client_rate_limiter::{ClientRateLimiter, RequestReason};
25use crate::client::retries::strategy::standard::ReleaseResult::{
26    APermitWasReleased, NoPermitWasReleased,
27};
28use crate::client::retries::token_bucket::TokenBucket;
29use crate::client::retries::{
30    ClientRateLimiterPartition, LongPollingBackoff, RetryPartition, RetryPartitionInner,
31};
32use crate::static_partition_map::StaticPartitionMap;
33
34static CLIENT_RATE_LIMITER: StaticPartitionMap<ClientRateLimiterPartition, ClientRateLimiter> =
35    StaticPartitionMap::new();
36
37/// Used by token bucket interceptor to ensure a TokenBucket always exists in config bag
38static TOKEN_BUCKET: StaticPartitionMap<RetryPartition, TokenBucket> = StaticPartitionMap::new();
39
40/// Retry strategy with exponential backoff, max attempts, and a token bucket.
41#[derive(Debug, Default)]
42pub struct StandardRetryStrategy {
43    retry_permit: Mutex<Option<OwnedSemaphorePermit>>,
44}
45
46impl Storable for StandardRetryStrategy {
47    type Storer = StoreReplace<Self>;
48}
49
50impl StandardRetryStrategy {
51    /// Create a new standard retry strategy with the given config.
52    pub fn new() -> Self {
53        Default::default()
54    }
55
56    fn release_retry_permit(&self, token_bucket: &TokenBucket) -> ReleaseResult {
57        let mut retry_permit = self.retry_permit.lock().unwrap();
58        match retry_permit.take() {
59            Some(p) => {
60                // Retry succeeded: reward success and forget permit if configured, otherwise release permit back
61                if token_bucket.success_reward() > 0.0 {
62                    token_bucket.reward_success();
63                    p.forget();
64                } else {
65                    drop(p); // Original behavior - release back to bucket
66                }
67                APermitWasReleased
68            }
69            None => {
70                // First-attempt success: reward success or regenerate token
71                if token_bucket.success_reward() > 0.0 {
72                    token_bucket.reward_success();
73                } else {
74                    token_bucket.regenerate_a_token();
75                }
76                NoPermitWasReleased
77            }
78        }
79    }
80
81    fn set_retry_permit(&self, new_retry_permit: OwnedSemaphorePermit) {
82        let mut old_retry_permit = self.retry_permit.lock().unwrap();
83        if let Some(p) = old_retry_permit.replace(new_retry_permit) {
84            // Whenever we set a new retry permit, and it replaces the old one, we need to "forget"
85            // the old permit, removing it from the bucket forever.
86            p.forget()
87        }
88    }
89
90    /// Returns a [`ClientRateLimiter`] if adaptive retry is configured.
91    fn adaptive_retry_rate_limiter(
92        runtime_components: &RuntimeComponents,
93        cfg: &ConfigBag,
94    ) -> Option<ClientRateLimiter> {
95        let retry_config = cfg.load::<RetryConfig>().expect("retry config is required");
96        if retry_config.mode() == RetryMode::Adaptive {
97            if let Some(time_source) = runtime_components.time_source() {
98                let retry_partition = cfg.load::<RetryPartition>().expect("set in default config");
99                let seconds_since_unix_epoch = time_source
100                    .now()
101                    .duration_since(SystemTime::UNIX_EPOCH)
102                    .expect("the present takes place after the UNIX_EPOCH")
103                    .as_secs_f64();
104                let client_rate_limiter = match &retry_partition.inner {
105                    RetryPartitionInner::Default(_) => {
106                        let client_rate_limiter_partition =
107                            ClientRateLimiterPartition::new(retry_partition.clone());
108                        CLIENT_RATE_LIMITER.get_or_init(client_rate_limiter_partition, || {
109                            ClientRateLimiter::new(seconds_since_unix_epoch)
110                        })
111                    }
112                    RetryPartitionInner::Custom {
113                        client_rate_limiter,
114                        ..
115                    } => client_rate_limiter.clone(),
116                };
117                return Some(client_rate_limiter);
118            }
119        }
120        None
121    }
122
123    fn calculate_backoff(
124        &self,
125        runtime_components: &RuntimeComponents,
126        cfg: &ConfigBag,
127        retry_cfg: &RetryConfig,
128        retry_reason: &RetryAction,
129    ) -> Result<Duration, ShouldAttempt> {
130        let request_attempts = cfg
131            .load::<RequestAttempts>()
132            .expect("at least one request attempt is made before any retry is attempted")
133            .attempts();
134
135        match retry_reason {
136            RetryAction::RetryIndicated(RetryReason::RetryableError { kind, retry_after }) => {
137                let initial_backoff = if *kind != ErrorKind::ThrottlingError {
138                    retry_cfg
139                        .retry_spec()
140                        .map(|s| s.non_throttling_initial_backoff())
141                        .unwrap_or(retry_cfg.initial_backoff())
142                        .as_secs_f64()
143                } else {
144                    retry_cfg.initial_backoff().as_secs_f64()
145                };
146
147                if let Some(delay) = check_rate_limiter_for_delay(runtime_components, cfg, *kind) {
148                    let delay = delay.min(retry_cfg.max_backoff());
149                    debug!("rate limiter has requested a {delay:?} delay before retrying");
150                    Ok(delay)
151                } else {
152                    let base = if retry_cfg.use_static_exponential_base() {
153                        1.0
154                    } else {
155                        fastrand::f64()
156                    };
157                    let t_i = calculate_exponential_backoff(
158                        base,
159                        initial_backoff,
160                        request_attempts - 1,
161                        retry_cfg.max_backoff(),
162                    );
163
164                    if let Some(retry_after) = *retry_after {
165                        if retry_cfg
166                            .retry_spec()
167                            .is_some_and(|s| s.is_at_least(RetrySpec::V2_1))
168                        {
169                            let delay = retry_after.clamp(t_i, t_i + Duration::from_secs(5));
170                            debug!("x-amz-retry-after bounded to {delay:?} (t_i={t_i:?})");
171                            Ok(delay)
172                        } else {
173                            let delay = retry_after.min(retry_cfg.max_backoff());
174                            debug!(
175                                "explicit request from server to delay {delay:?} before retrying"
176                            );
177                            Ok(delay)
178                        }
179                    } else {
180                        Ok(t_i)
181                    }
182                }
183            }
184            RetryAction::RetryForbidden | RetryAction::NoActionIndicated => {
185                debug!(
186                    attempts = request_attempts,
187                    max_attempts = retry_cfg.max_attempts(),
188                    "encountered un-retryable error"
189                );
190                Err(ShouldAttempt::No)
191            }
192            _ => unreachable!("RetryAction is non-exhaustive"),
193        }
194    }
195}
196
197enum ReleaseResult {
198    APermitWasReleased,
199    NoPermitWasReleased,
200}
201
202impl RetryStrategy for StandardRetryStrategy {
203    fn should_attempt_initial_request(
204        &self,
205        runtime_components: &RuntimeComponents,
206        cfg: &ConfigBag,
207    ) -> Result<ShouldAttempt, BoxError> {
208        if let Some(crl) = Self::adaptive_retry_rate_limiter(runtime_components, cfg) {
209            let seconds_since_unix_epoch = get_seconds_since_unix_epoch(runtime_components);
210            if let Err(delay) = crl.acquire_permission_to_send_a_request(
211                seconds_since_unix_epoch,
212                RequestReason::InitialRequest,
213            ) {
214                return Ok(ShouldAttempt::YesAfterDelay(delay));
215            }
216        } else {
217            debug!("no client rate limiter configured, so no token is required for the initial request.");
218        }
219
220        Ok(ShouldAttempt::Yes)
221    }
222
223    fn should_attempt_retry(
224        &self,
225        ctx: &InterceptorContext,
226        runtime_components: &RuntimeComponents,
227        cfg: &ConfigBag,
228    ) -> Result<ShouldAttempt, BoxError> {
229        let retry_cfg = cfg.load::<RetryConfig>().expect("retry config is required");
230
231        // bookkeeping
232        let token_bucket = cfg.load::<TokenBucket>().expect("token bucket is required");
233        // run the classifier against the context to determine if we should retry
234        let retry_classifiers = runtime_components.retry_classifiers();
235        let classifier_result = run_classifiers_on_ctx(retry_classifiers, ctx);
236
237        // (adaptive only): update fill rate
238        // NOTE: the retry spec indicates doing bookkeeping before asking if we should retry. We need to know if
239        // the error was a throttling error though to do adaptive retry bookkeeping so we take
240        // advantage of that information being available via the classifier result
241        let error_kind = error_kind(&classifier_result);
242        let is_throttling_error = error_kind
243            .map(|kind| kind == ErrorKind::ThrottlingError)
244            .unwrap_or(false);
245        update_rate_limiter_if_exists(runtime_components, cfg, is_throttling_error);
246
247        // on success release any retry quota held by previous attempts, reward success when indicated
248        if !ctx.is_failed() {
249            self.release_retry_permit(token_bucket);
250        }
251        // end bookkeeping
252
253        let request_attempts = cfg
254            .load::<RequestAttempts>()
255            .expect("at least one request attempt is made before any retry is attempted")
256            .attempts();
257
258        // check if retry should be attempted
259        if !classifier_result.should_retry() {
260            debug!(
261                "attempt #{request_attempts} classified as {:?}, not retrying",
262                classifier_result
263            );
264            return Ok(ShouldAttempt::No);
265        }
266
267        // check if we're out of attempts
268        if request_attempts >= retry_cfg.max_attempts() {
269            debug!(
270                attempts = request_attempts,
271                max_attempts = retry_cfg.max_attempts(),
272                "not retrying because we are out of attempts"
273            );
274            return Ok(ShouldAttempt::No);
275        }
276
277        //  acquire permit for retry
278        let error_kind = error_kind.expect("result was classified retryable");
279        let is_long_polling = retry_cfg.retry_spec().is_some_and(|s| s.long_polling());
280
281        // Calculate backoff before token check. For long-polling services, this ensures
282        // the caller's loop is slowed down even when the token bucket is empty.
283        let backoff =
284            match self.calculate_backoff(runtime_components, cfg, retry_cfg, &classifier_result) {
285                Ok(value) => value,
286                Err(value) => return Ok(value),
287            };
288
289        //  acquire permit for retry
290        match token_bucket.acquire(
291            &error_kind,
292            &runtime_components.time_source().unwrap_or_default(),
293        ) {
294            Some(permit) => self.set_retry_permit(permit),
295            None => {
296                debug!("attempt #{request_attempts} failed with {error_kind:?}; not enough retry quota.");
297                if is_long_polling {
298                    if let Some(hint) = cfg.load::<LongPollingBackoff>() {
299                        hint.set(backoff);
300                    }
301                }
302                return Ok(ShouldAttempt::No);
303            }
304        }
305
306        debug!(
307            "attempt #{request_attempts} failed with {:?}; retrying after {:?}",
308            classifier_result, backoff
309        );
310        Ok(ShouldAttempt::YesAfterDelay(backoff))
311    }
312}
313
314/// extract the error kind from the classifier result if available
315fn error_kind(classifier_result: &RetryAction) -> Option<ErrorKind> {
316    match classifier_result {
317        RetryAction::RetryIndicated(RetryReason::RetryableError { kind, .. }) => Some(*kind),
318        _ => None,
319    }
320}
321
322fn update_rate_limiter_if_exists(
323    runtime_components: &RuntimeComponents,
324    cfg: &ConfigBag,
325    is_throttling_error: bool,
326) {
327    if let Some(crl) = StandardRetryStrategy::adaptive_retry_rate_limiter(runtime_components, cfg) {
328        let seconds_since_unix_epoch = get_seconds_since_unix_epoch(runtime_components);
329        crl.update_rate_limiter(seconds_since_unix_epoch, is_throttling_error);
330    }
331}
332
333fn check_rate_limiter_for_delay(
334    runtime_components: &RuntimeComponents,
335    cfg: &ConfigBag,
336    kind: ErrorKind,
337) -> Option<Duration> {
338    if let Some(crl) = StandardRetryStrategy::adaptive_retry_rate_limiter(runtime_components, cfg) {
339        // Retry Behavior 2.1 acquires one adaptive send token per attempt in the
340        // orchestrator's send loop (GetSendToken: sleep-then-re-acquire, so
341        // capacity is never driven negative). The rate limiter therefore must
342        // NOT also fold an acquire delay into the retry backoff here; the
343        // x-amz-retry-after / exponential backoff is applied on its own below.
344        // Pre-2.1 keeps the legacy behavior of folding the acquire delay (with
345        // the 5/10-token retry costs) into the backoff.
346        let is_v2_1 = cfg
347            .load::<RetryConfig>()
348            .and_then(|rc| rc.retry_spec())
349            .is_some_and(|s| s.is_at_least(RetrySpec::V2_1));
350        if is_v2_1 {
351            return None;
352        }
353        let retry_reason = if kind == ErrorKind::ThrottlingError {
354            RequestReason::RetryTimeout
355        } else {
356            RequestReason::Retry
357        };
358        if let Err(delay) = crl.acquire_permission_to_send_a_request(
359            get_seconds_since_unix_epoch(runtime_components),
360            retry_reason,
361        ) {
362            return Some(delay);
363        }
364    }
365
366    None
367}
368
369pub(super) fn calculate_exponential_backoff(
370    base: f64,
371    initial_backoff: f64,
372    retry_attempts: u32,
373    max_backoff: Duration,
374) -> Duration {
375    let result = match 2_u32
376        .checked_pow(retry_attempts)
377        .map(|power| (power as f64) * initial_backoff)
378    {
379        Some(backoff) => match Duration::try_from_secs_f64(backoff) {
380            Ok(result) => result.min(max_backoff),
381            Err(e) => {
382                tracing::warn!("falling back to {max_backoff:?} as `Duration` could not be created for exponential backoff: {e}");
383                max_backoff
384            }
385        },
386        None => max_backoff,
387    };
388
389    // Apply jitter to `result`, and note that it can be applied to `max_backoff`.
390    // Won't panic because `base` is either in range 0..1 or a constant 1 in testing (if configured).
391    result.mul_f64(base)
392}
393
394pub(super) fn get_seconds_since_unix_epoch(runtime_components: &RuntimeComponents) -> f64 {
395    let request_time = runtime_components
396        .time_source()
397        .expect("time source required for retries");
398    request_time
399        .now()
400        .duration_since(SystemTime::UNIX_EPOCH)
401        .unwrap()
402        .as_secs_f64()
403}
404
405/// Interceptor registered in default retry plugin that ensures a token bucket exists in config
406/// bag for every operation. Token bucket provided is partitioned by the retry partition **in the
407/// config bag** at the time an operation is executed.
408#[derive(Debug)]
409pub(crate) struct TokenBucketProvider {
410    default_partition: RetryPartition,
411    token_bucket: OnceLock<TokenBucket>,
412}
413
414impl TokenBucketProvider {
415    /// Create a new token bucket provider with the given default retry partition.
416    ///
417    /// NOTE: This partition should be the one used for every operation on a client
418    /// unless config is overridden.
419    pub(crate) fn new(default_partition: RetryPartition) -> Self {
420        Self {
421            default_partition,
422            token_bucket: OnceLock::new(),
423        }
424    }
425}
426
427/// Build a token bucket with costs determined by the RetrySpec in the config bag.
428fn token_bucket_for_spec(cfg: &ConfigBag) -> TokenBucket {
429    let is_v2_1 = cfg
430        .load::<RetryConfig>()
431        .and_then(|rc| rc.retry_spec())
432        .is_some_and(|s| s.is_at_least(RetrySpec::V2_1));
433    if is_v2_1 {
434        TokenBucket::builder()
435            .retry_cost(14)
436            .throttling_retry_cost(5)
437            .timeout_retry_cost(14)
438            .build()
439    } else {
440        TokenBucket::default()
441    }
442}
443
444#[dyn_dispatch_hint]
445impl Intercept for TokenBucketProvider {
446    fn name(&self) -> &'static str {
447        "TokenBucketProvider"
448    }
449
450    fn modify_before_retry_loop(
451        &self,
452        _context: &mut BeforeTransmitInterceptorContextMut<'_>,
453        _runtime_components: &RuntimeComponents,
454        cfg: &mut ConfigBag,
455    ) -> Result<(), BoxError> {
456        let retry_partition = cfg.load::<RetryPartition>().expect("set in default config");
457
458        let tb = match &retry_partition.inner {
459            RetryPartitionInner::Default(name) => {
460                if name == self.default_partition.name() {
461                    self.token_bucket
462                        .get_or_init(|| {
463                            TOKEN_BUCKET.get_or_init(self.default_partition.clone(), || {
464                                token_bucket_for_spec(cfg)
465                            })
466                        })
467                        .clone()
468                } else {
469                    TOKEN_BUCKET.get_or_init(retry_partition.clone(), || token_bucket_for_spec(cfg))
470                }
471            }
472            RetryPartitionInner::Custom { token_bucket, .. } => token_bucket.clone(),
473        };
474
475        trace!("token bucket for {retry_partition:?} added to config bag");
476        let mut layer = Layer::new("token_bucket_partition");
477        layer.store_put(tb);
478        cfg.push_layer(layer);
479        Ok(())
480    }
481}
482
483#[cfg(test)]
484mod tests {
485    #[allow(unused_imports)] // will be unused with `--no-default-features --features client`
486    use std::fmt;
487    use std::sync::Mutex;
488    use std::time::Duration;
489
490    use aws_smithy_async::time::SystemTimeSource;
491    use aws_smithy_runtime_api::client::interceptors::context::{
492        Input, InterceptorContext, Output,
493    };
494    use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
495    use aws_smithy_runtime_api::client::retries::classifiers::{
496        ClassifyRetry, RetryAction, SharedRetryClassifier,
497    };
498    use aws_smithy_runtime_api::client::retries::{
499        AlwaysRetry, RequestAttempts, RetryStrategy, ShouldAttempt,
500    };
501    use aws_smithy_runtime_api::client::runtime_components::{
502        RuntimeComponents, RuntimeComponentsBuilder,
503    };
504    use aws_smithy_types::config_bag::{ConfigBag, Layer};
505    use aws_smithy_types::retry::{ErrorKind, RetryConfig};
506
507    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
508    use aws_smithy_types::retry::RetrySpec;
509
510    use super::{calculate_exponential_backoff, StandardRetryStrategy};
511    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
512    use crate::client::retries::token_bucket::{
513        DEFAULT_CAPACITY, DEFAULT_RETRY_COST, DEFAULT_RETRY_TIMEOUT_COST, THROTTLING_RETRY_COST,
514    };
515    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
516    use crate::client::retries::LongPollingBackoff;
517    use crate::client::retries::{ClientRateLimiter, RetryPartition, TokenBucket};
518
519    #[test]
520    fn no_retry_necessary_for_ok_result() {
521        let cfg = ConfigBag::of_layers(vec![{
522            let mut layer = Layer::new("test");
523            layer.store_put(RetryConfig::standard());
524            layer.store_put(RequestAttempts::new(1));
525            layer.store_put(TokenBucket::default());
526            layer
527        }]);
528        let rc = RuntimeComponentsBuilder::for_tests().build().unwrap();
529        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
530        let strategy = StandardRetryStrategy::default();
531        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
532
533        let actual = strategy
534            .should_attempt_retry(&ctx, &rc, &cfg)
535            .expect("method is infallible for this use");
536        assert_eq!(ShouldAttempt::No, actual);
537    }
538
539    fn set_up_cfg_and_context(
540        error_kind: ErrorKind,
541        current_request_attempts: u32,
542        retry_config: RetryConfig,
543    ) -> (InterceptorContext, RuntimeComponents, ConfigBag) {
544        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
545        ctx.set_output_or_error(Err(OrchestratorError::other("doesn't matter")));
546        let rc = RuntimeComponentsBuilder::for_tests()
547            .with_retry_classifier(SharedRetryClassifier::new(AlwaysRetry(error_kind)))
548            .build()
549            .unwrap();
550        let mut layer = Layer::new("test");
551        layer.store_put(RequestAttempts::new(current_request_attempts));
552        layer.store_put(retry_config);
553        layer.store_put(TokenBucket::default());
554        let cfg = ConfigBag::of_layers(vec![layer]);
555
556        (ctx, rc, cfg)
557    }
558
559    // Test that error kinds produce the correct "retry after X seconds" output.
560    // All error kinds are handled in the same way for the standard strategy.
561    fn test_should_retry_error_kind(error_kind: ErrorKind) {
562        let (ctx, rc, cfg) = set_up_cfg_and_context(
563            error_kind,
564            3,
565            RetryConfig::standard()
566                .with_use_static_exponential_base(true)
567                .with_max_attempts(4),
568        );
569        let strategy = StandardRetryStrategy::new();
570        let actual = strategy
571            .should_attempt_retry(&ctx, &rc, &cfg)
572            .expect("method is infallible for this use");
573        assert_eq!(ShouldAttempt::YesAfterDelay(Duration::from_secs(4)), actual);
574    }
575
576    #[test]
577    fn should_retry_transient_error_result_after_2s() {
578        test_should_retry_error_kind(ErrorKind::TransientError);
579    }
580
581    #[test]
582    fn should_retry_client_error_result_after_2s() {
583        test_should_retry_error_kind(ErrorKind::ClientError);
584    }
585
586    #[test]
587    fn should_retry_server_error_result_after_2s() {
588        test_should_retry_error_kind(ErrorKind::ServerError);
589    }
590
591    #[test]
592    fn should_retry_throttling_error_result_after_2s() {
593        test_should_retry_error_kind(ErrorKind::ThrottlingError);
594    }
595
596    #[test]
597    fn dont_retry_when_out_of_attempts() {
598        let current_attempts = 4;
599        let max_attempts = current_attempts;
600        let (ctx, rc, cfg) = set_up_cfg_and_context(
601            ErrorKind::TransientError,
602            current_attempts,
603            RetryConfig::standard()
604                .with_use_static_exponential_base(true)
605                .with_max_attempts(max_attempts),
606        );
607        let strategy = StandardRetryStrategy::new();
608        let actual = strategy
609            .should_attempt_retry(&ctx, &rc, &cfg)
610            .expect("method is infallible for this use");
611        assert_eq!(ShouldAttempt::No, actual);
612    }
613
614    #[test]
615    fn should_not_panic_when_exponential_backoff_duration_could_not_be_created() {
616        let (ctx, rc, cfg) = set_up_cfg_and_context(
617            ErrorKind::TransientError,
618            // Greater than 32 when subtracted by 1 in `calculate_backoff`, causing overflow in `calculate_exponential_backoff`
619            33,
620            RetryConfig::standard()
621                .with_use_static_exponential_base(true)
622                .with_max_attempts(100), // Any value greater than 33 will do
623        );
624        let strategy = StandardRetryStrategy::new();
625        let actual = strategy
626            .should_attempt_retry(&ctx, &rc, &cfg)
627            .expect("method is infallible for this use");
628        assert_eq!(ShouldAttempt::YesAfterDelay(MAX_BACKOFF), actual);
629    }
630
631    #[test]
632    fn should_yield_client_rate_limiter_from_custom_partition() {
633        let expected = ClientRateLimiter::builder().token_refill_rate(3.14).build();
634        let cfg = ConfigBag::of_layers(vec![
635            // Emulate default config layer overriden by a user config layer
636            {
637                let mut layer = Layer::new("default");
638                layer.store_put(RetryPartition::new("default"));
639                layer
640            },
641            {
642                let mut layer = Layer::new("user");
643                layer.store_put(RetryConfig::adaptive());
644                layer.store_put(
645                    RetryPartition::custom("user")
646                        .client_rate_limiter(expected.clone())
647                        .build(),
648                );
649                layer
650            },
651        ]);
652        let rc = RuntimeComponentsBuilder::for_tests()
653            .with_time_source(Some(SystemTimeSource::new()))
654            .build()
655            .unwrap();
656        let actual = StandardRetryStrategy::adaptive_retry_rate_limiter(&rc, &cfg)
657            .expect("should yield client rate limiter from custom partition");
658        assert!(std::sync::Arc::ptr_eq(&expected.inner, &actual.inner));
659    }
660
661    #[allow(dead_code)] // will be unused with `--no-default-features --features client`
662    #[derive(Debug)]
663    struct PresetReasonRetryClassifier {
664        retry_actions: Mutex<Vec<RetryAction>>,
665    }
666
667    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
668    impl PresetReasonRetryClassifier {
669        fn new(mut retry_reasons: Vec<RetryAction>) -> Self {
670            // We'll pop the retry_reasons in reverse order, so we reverse the list to fix that.
671            retry_reasons.reverse();
672            Self {
673                retry_actions: Mutex::new(retry_reasons),
674            }
675        }
676    }
677
678    impl ClassifyRetry for PresetReasonRetryClassifier {
679        fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
680            // Check for a result
681            let output_or_error = ctx.output_or_error();
682            // Check for an error
683            match output_or_error {
684                Some(Ok(_)) | None => return RetryAction::NoActionIndicated,
685                _ => (),
686            };
687
688            let mut retry_actions = self.retry_actions.lock().unwrap();
689            if retry_actions.len() == 1 {
690                retry_actions.first().unwrap().clone()
691            } else {
692                retry_actions.pop().unwrap()
693            }
694        }
695
696        fn name(&self) -> &'static str {
697            "Always returns a preset retry reason"
698        }
699    }
700
701    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
702    fn setup_test(
703        retry_reasons: Vec<RetryAction>,
704        retry_config: RetryConfig,
705    ) -> (ConfigBag, RuntimeComponents, InterceptorContext) {
706        let rc = RuntimeComponentsBuilder::for_tests()
707            .with_retry_classifier(SharedRetryClassifier::new(
708                PresetReasonRetryClassifier::new(retry_reasons),
709            ))
710            .build()
711            .unwrap();
712        let mut layer = Layer::new("test");
713        layer.store_put(retry_config);
714        let cfg = ConfigBag::of_layers(vec![layer]);
715        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
716        // This type doesn't matter b/c the classifier will just return whatever we tell it to.
717        ctx.set_output_or_error(Err(OrchestratorError::other("doesn't matter")));
718
719        (cfg, rc, ctx)
720    }
721
722    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
723    #[test]
724    fn eventual_success() {
725        let (mut cfg, rc, mut ctx) = setup_test(
726            vec![RetryAction::server_error()],
727            RetryConfig::standard()
728                .with_use_static_exponential_base(true)
729                .with_max_attempts(5),
730        );
731        let strategy = StandardRetryStrategy::new();
732        cfg.interceptor_state().store_put(TokenBucket::default());
733        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
734
735        cfg.interceptor_state().store_put(RequestAttempts::new(1));
736        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
737        let dur = should_retry.expect_delay();
738        assert_eq!(dur, Duration::from_secs(1));
739        assert_eq!(token_bucket.available_permits(), 495);
740
741        cfg.interceptor_state().store_put(RequestAttempts::new(2));
742        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
743        let dur = should_retry.expect_delay();
744        assert_eq!(dur, Duration::from_secs(2));
745        assert_eq!(token_bucket.available_permits(), 490);
746
747        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
748
749        cfg.interceptor_state().store_put(RequestAttempts::new(3));
750        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
751        assert_eq!(no_retry, ShouldAttempt::No);
752        assert_eq!(token_bucket.available_permits(), 495);
753    }
754
755    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
756    #[test]
757    fn no_more_attempts() {
758        let (mut cfg, rc, ctx) = setup_test(
759            vec![RetryAction::server_error()],
760            RetryConfig::standard()
761                .with_use_static_exponential_base(true)
762                .with_max_attempts(3),
763        );
764        let strategy = StandardRetryStrategy::new();
765        cfg.interceptor_state().store_put(TokenBucket::default());
766        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
767
768        cfg.interceptor_state().store_put(RequestAttempts::new(1));
769        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
770        let dur = should_retry.expect_delay();
771        assert_eq!(dur, Duration::from_secs(1));
772        assert_eq!(token_bucket.available_permits(), 495);
773
774        cfg.interceptor_state().store_put(RequestAttempts::new(2));
775        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
776        let dur = should_retry.expect_delay();
777        assert_eq!(dur, Duration::from_secs(2));
778        assert_eq!(token_bucket.available_permits(), 490);
779
780        cfg.interceptor_state().store_put(RequestAttempts::new(3));
781        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
782        assert_eq!(no_retry, ShouldAttempt::No);
783        assert_eq!(token_bucket.available_permits(), 490);
784    }
785
786    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
787    #[test]
788    fn successful_request_and_deser_should_be_retryable() {
789        #[derive(Clone, Copy, Debug)]
790        enum LongRunningOperationStatus {
791            Running,
792            Complete,
793        }
794
795        #[derive(Debug)]
796        struct LongRunningOperationOutput {
797            status: Option<LongRunningOperationStatus>,
798        }
799
800        impl LongRunningOperationOutput {
801            fn status(&self) -> Option<LongRunningOperationStatus> {
802                self.status
803            }
804        }
805
806        struct WaiterRetryClassifier {}
807
808        impl WaiterRetryClassifier {
809            fn new() -> Self {
810                WaiterRetryClassifier {}
811            }
812        }
813
814        impl fmt::Debug for WaiterRetryClassifier {
815            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
816                write!(f, "WaiterRetryClassifier")
817            }
818        }
819        impl ClassifyRetry for WaiterRetryClassifier {
820            fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
821                let status: Option<LongRunningOperationStatus> =
822                    ctx.output_or_error().and_then(|res| {
823                        res.ok().and_then(|output| {
824                            output
825                                .downcast_ref::<LongRunningOperationOutput>()
826                                .and_then(|output| output.status())
827                        })
828                    });
829
830                if let Some(LongRunningOperationStatus::Running) = status {
831                    return RetryAction::server_error();
832                };
833
834                RetryAction::NoActionIndicated
835            }
836
837            fn name(&self) -> &'static str {
838                "waiter retry classifier"
839            }
840        }
841
842        let retry_config = RetryConfig::standard()
843            .with_use_static_exponential_base(true)
844            .with_max_attempts(5);
845
846        let rc = RuntimeComponentsBuilder::for_tests()
847            .with_retry_classifier(SharedRetryClassifier::new(WaiterRetryClassifier::new()))
848            .build()
849            .unwrap();
850        let mut layer = Layer::new("test");
851        layer.store_put(retry_config);
852        let mut cfg = ConfigBag::of_layers(vec![layer]);
853        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
854        let strategy = StandardRetryStrategy::new();
855
856        ctx.set_output_or_error(Ok(Output::erase(LongRunningOperationOutput {
857            status: Some(LongRunningOperationStatus::Running),
858        })));
859
860        cfg.interceptor_state().store_put(TokenBucket::new(5));
861        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
862
863        cfg.interceptor_state().store_put(RequestAttempts::new(1));
864        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
865        let dur = should_retry.expect_delay();
866        assert_eq!(dur, Duration::from_secs(1));
867        assert_eq!(token_bucket.available_permits(), 0);
868
869        ctx.set_output_or_error(Ok(Output::erase(LongRunningOperationOutput {
870            status: Some(LongRunningOperationStatus::Complete),
871        })));
872        cfg.interceptor_state().store_put(RequestAttempts::new(2));
873        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
874        should_retry.expect_no();
875        assert_eq!(token_bucket.available_permits(), 5);
876    }
877
878    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
879    #[test]
880    fn no_quota() {
881        let (mut cfg, rc, ctx) = setup_test(
882            vec![RetryAction::server_error()],
883            RetryConfig::standard()
884                .with_use_static_exponential_base(true)
885                .with_max_attempts(5),
886        );
887        let strategy = StandardRetryStrategy::new();
888        cfg.interceptor_state().store_put(TokenBucket::new(5));
889        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
890
891        cfg.interceptor_state().store_put(RequestAttempts::new(1));
892        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
893        let dur = should_retry.expect_delay();
894        assert_eq!(dur, Duration::from_secs(1));
895        assert_eq!(token_bucket.available_permits(), 0);
896
897        cfg.interceptor_state().store_put(RequestAttempts::new(2));
898        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
899        assert_eq!(no_retry, ShouldAttempt::No);
900        assert_eq!(token_bucket.available_permits(), 0);
901    }
902
903    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
904    #[test]
905    fn quota_replenishes_on_success() {
906        let (mut cfg, rc, mut ctx) = setup_test(
907            vec![
908                RetryAction::transient_error(),
909                RetryAction::retryable_error_with_explicit_delay(
910                    ErrorKind::TransientError,
911                    Duration::from_secs(1),
912                ),
913            ],
914            RetryConfig::standard()
915                .with_use_static_exponential_base(true)
916                .with_max_attempts(5),
917        );
918        let strategy = StandardRetryStrategy::new();
919        cfg.interceptor_state().store_put(TokenBucket::new(100));
920        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
921
922        cfg.interceptor_state().store_put(RequestAttempts::new(1));
923        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
924        let dur = should_retry.expect_delay();
925        assert_eq!(dur, Duration::from_secs(1));
926        assert_eq!(token_bucket.available_permits(), 90);
927
928        cfg.interceptor_state().store_put(RequestAttempts::new(2));
929        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
930        let dur = should_retry.expect_delay();
931        assert_eq!(dur, Duration::from_secs(1));
932        assert_eq!(token_bucket.available_permits(), 80);
933
934        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
935
936        cfg.interceptor_state().store_put(RequestAttempts::new(3));
937        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
938        assert_eq!(no_retry, ShouldAttempt::No);
939
940        assert_eq!(token_bucket.available_permits(), 90);
941    }
942
943    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
944    #[test]
945    fn quota_replenishes_on_first_try_success() {
946        const PERMIT_COUNT: usize = 20;
947        let (mut cfg, rc, mut ctx) = setup_test(
948            vec![RetryAction::transient_error()],
949            RetryConfig::standard()
950                .with_use_static_exponential_base(true)
951                .with_max_attempts(u32::MAX),
952        );
953        let strategy = StandardRetryStrategy::new();
954        cfg.interceptor_state()
955            .store_put(TokenBucket::new(PERMIT_COUNT));
956        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
957
958        let mut attempt = 1;
959
960        // Drain all available permits with failed attempts
961        while token_bucket.available_permits() > 0 {
962            // Draining should complete in 2 attempts
963            if attempt > 2 {
964                panic!("This test should have completed by now (drain)");
965            }
966
967            cfg.interceptor_state()
968                .store_put(RequestAttempts::new(attempt));
969            let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
970            assert!(matches!(should_retry, ShouldAttempt::YesAfterDelay(_)));
971            attempt += 1;
972        }
973
974        // Forget the permit so that we can only refill by "success on first try".
975        let permit = strategy.retry_permit.lock().unwrap().take().unwrap();
976        permit.forget();
977
978        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
979
980        // Replenish permits until we get back to `PERMIT_COUNT`
981        while token_bucket.available_permits() < PERMIT_COUNT {
982            if attempt > 23 {
983                panic!("This test should have completed by now (fill-up)");
984            }
985
986            cfg.interceptor_state()
987                .store_put(RequestAttempts::new(attempt));
988            let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
989            assert_eq!(no_retry, ShouldAttempt::No);
990            attempt += 1;
991        }
992
993        assert_eq!(attempt, 23);
994        assert_eq!(token_bucket.available_permits(), PERMIT_COUNT);
995    }
996
997    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
998    #[test]
999    fn backoff_timing() {
1000        let (mut cfg, rc, ctx) = setup_test(
1001            vec![RetryAction::server_error()],
1002            RetryConfig::standard()
1003                .with_use_static_exponential_base(true)
1004                .with_max_attempts(5),
1005        );
1006        let strategy = StandardRetryStrategy::new();
1007        cfg.interceptor_state().store_put(TokenBucket::default());
1008        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
1009
1010        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1011        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1012        let dur = should_retry.expect_delay();
1013        assert_eq!(dur, Duration::from_secs(1));
1014        assert_eq!(token_bucket.available_permits(), 495);
1015
1016        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1017        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1018        let dur = should_retry.expect_delay();
1019        assert_eq!(dur, Duration::from_secs(2));
1020        assert_eq!(token_bucket.available_permits(), 490);
1021
1022        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1023        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1024        let dur = should_retry.expect_delay();
1025        assert_eq!(dur, Duration::from_secs(4));
1026        assert_eq!(token_bucket.available_permits(), 485);
1027
1028        cfg.interceptor_state().store_put(RequestAttempts::new(4));
1029        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1030        let dur = should_retry.expect_delay();
1031        assert_eq!(dur, Duration::from_secs(8));
1032        assert_eq!(token_bucket.available_permits(), 480);
1033
1034        cfg.interceptor_state().store_put(RequestAttempts::new(5));
1035        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1036        assert_eq!(no_retry, ShouldAttempt::No);
1037        assert_eq!(token_bucket.available_permits(), 480);
1038    }
1039
1040    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1041    #[test]
1042    fn max_backoff_time() {
1043        let (mut cfg, rc, ctx) = setup_test(
1044            vec![RetryAction::server_error()],
1045            RetryConfig::standard()
1046                .with_use_static_exponential_base(true)
1047                .with_max_attempts(5)
1048                .with_initial_backoff(Duration::from_secs(1))
1049                .with_max_backoff(Duration::from_secs(3)),
1050        );
1051        let strategy = StandardRetryStrategy::new();
1052        cfg.interceptor_state().store_put(TokenBucket::default());
1053        let token_bucket = cfg.load::<TokenBucket>().unwrap().clone();
1054
1055        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1056        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1057        let dur = should_retry.expect_delay();
1058        assert_eq!(dur, Duration::from_secs(1));
1059        assert_eq!(token_bucket.available_permits(), 495);
1060
1061        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1062        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1063        let dur = should_retry.expect_delay();
1064        assert_eq!(dur, Duration::from_secs(2));
1065        assert_eq!(token_bucket.available_permits(), 490);
1066
1067        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1068        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1069        let dur = should_retry.expect_delay();
1070        assert_eq!(dur, Duration::from_secs(3));
1071        assert_eq!(token_bucket.available_permits(), 485);
1072
1073        cfg.interceptor_state().store_put(RequestAttempts::new(4));
1074        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1075        let dur = should_retry.expect_delay();
1076        assert_eq!(dur, Duration::from_secs(3));
1077        assert_eq!(token_bucket.available_permits(), 480);
1078
1079        cfg.interceptor_state().store_put(RequestAttempts::new(5));
1080        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1081        assert_eq!(no_retry, ShouldAttempt::No);
1082        assert_eq!(token_bucket.available_permits(), 480);
1083    }
1084
1085    const MAX_BACKOFF: Duration = Duration::from_secs(20);
1086
1087    #[test]
1088    fn calculate_exponential_backoff_where_initial_backoff_is_one() {
1089        let initial_backoff = 1.0;
1090
1091        for (attempt, expected_backoff) in [initial_backoff, 2.0, 4.0].into_iter().enumerate() {
1092            let actual_backoff =
1093                calculate_exponential_backoff(1.0, initial_backoff, attempt as u32, MAX_BACKOFF);
1094            assert_eq!(Duration::from_secs_f64(expected_backoff), actual_backoff);
1095        }
1096    }
1097
1098    #[test]
1099    fn calculate_exponential_backoff_where_initial_backoff_is_greater_than_one() {
1100        let initial_backoff = 3.0;
1101
1102        for (attempt, expected_backoff) in [initial_backoff, 6.0, 12.0].into_iter().enumerate() {
1103            let actual_backoff =
1104                calculate_exponential_backoff(1.0, initial_backoff, attempt as u32, MAX_BACKOFF);
1105            assert_eq!(Duration::from_secs_f64(expected_backoff), actual_backoff);
1106        }
1107    }
1108
1109    #[test]
1110    fn calculate_exponential_backoff_where_initial_backoff_is_less_than_one() {
1111        let initial_backoff = 0.03;
1112
1113        for (attempt, expected_backoff) in [initial_backoff, 0.06, 0.12].into_iter().enumerate() {
1114            let actual_backoff =
1115                calculate_exponential_backoff(1.0, initial_backoff, attempt as u32, MAX_BACKOFF);
1116            assert_eq!(Duration::from_secs_f64(expected_backoff), actual_backoff);
1117        }
1118    }
1119
1120    #[test]
1121    fn calculate_backoff_overflow_should_gracefully_fallback_to_max_backoff() {
1122        // avoid overflow for a silly large amount of retry attempts
1123        assert_eq!(
1124            MAX_BACKOFF,
1125            calculate_exponential_backoff(1_f64, 10_f64, 100000, MAX_BACKOFF),
1126        );
1127    }
1128
1129    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1130    #[test]
1131    fn v2_1_non_throttling_uses_50ms_backoff() {
1132        let (ctx, rc, cfg) = set_up_cfg_and_context(
1133            ErrorKind::ServerError,
1134            1,
1135            RetryConfig::standard()
1136                .with_use_static_exponential_base(true)
1137                .with_max_attempts(3)
1138                .with_retry_spec(RetrySpec::v2_1()),
1139        );
1140        let strategy = StandardRetryStrategy::new();
1141        let actual = strategy
1142            .should_attempt_retry(&ctx, &rc, &cfg)
1143            .expect("method is infallible for this use");
1144        // 50ms * 2^0 = 50ms
1145        assert_eq!(
1146            ShouldAttempt::YesAfterDelay(Duration::from_millis(50)),
1147            actual
1148        );
1149    }
1150
1151    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1152    #[test]
1153    fn v2_1_throttling_uses_1s_backoff() {
1154        let (ctx, rc, cfg) = set_up_cfg_and_context(
1155            ErrorKind::ThrottlingError,
1156            1,
1157            RetryConfig::standard()
1158                .with_use_static_exponential_base(true)
1159                .with_max_attempts(3)
1160                .with_retry_spec(RetrySpec::v2_1()),
1161        );
1162        let strategy = StandardRetryStrategy::new();
1163        let actual = strategy
1164            .should_attempt_retry(&ctx, &rc, &cfg)
1165            .expect("method is infallible for this use");
1166        // 1s * 2^0 = 1s (throttling keeps legacy backoff)
1167        assert_eq!(ShouldAttempt::YesAfterDelay(Duration::from_secs(1)), actual);
1168    }
1169
1170    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1171    #[test]
1172    fn v2_0_non_throttling_uses_1s_backoff() {
1173        let (ctx, rc, cfg) = set_up_cfg_and_context(
1174            ErrorKind::ServerError,
1175            1,
1176            RetryConfig::standard()
1177                .with_use_static_exponential_base(true)
1178                .with_max_attempts(3)
1179                .with_retry_spec(RetrySpec::v2_0()),
1180        );
1181        let strategy = StandardRetryStrategy::new();
1182        let actual = strategy
1183            .should_attempt_retry(&ctx, &rc, &cfg)
1184            .expect("method is infallible for this use");
1185        // 1s * 2^0 = 1s (v2.0 keeps legacy backoff)
1186        assert_eq!(ShouldAttempt::YesAfterDelay(Duration::from_secs(1)), actual);
1187    }
1188
1189    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1190    #[test]
1191    fn v2_1_retry_after_bounded_between_t_i_and_t_i_plus_5s() {
1192        let (mut cfg, rc, ctx) = setup_test(
1193            vec![RetryAction::retryable_error_with_explicit_delay(
1194                ErrorKind::ServerError,
1195                Duration::from_secs(3),
1196            )],
1197            RetryConfig::standard()
1198                .with_use_static_exponential_base(true)
1199                .with_max_attempts(3)
1200                .with_retry_spec(RetrySpec::v2_1()),
1201        );
1202        let strategy = StandardRetryStrategy::new();
1203        cfg.interceptor_state().store_put(TokenBucket::default());
1204        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1205        let actual = strategy
1206            .should_attempt_retry(&ctx, &rc, &cfg)
1207            .expect("method is infallible for this use");
1208        // t_i = 50ms, retry_after = 3s, clamp(3s, 50ms, 5.05s) = 3s
1209        assert_eq!(ShouldAttempt::YesAfterDelay(Duration::from_secs(3)), actual);
1210    }
1211
1212    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1213    #[test]
1214    fn v2_1_retry_after_below_t_i_uses_t_i() {
1215        let (mut cfg, rc, ctx) = setup_test(
1216            vec![RetryAction::retryable_error_with_explicit_delay(
1217                ErrorKind::ServerError,
1218                Duration::from_millis(10),
1219            )],
1220            RetryConfig::standard()
1221                .with_use_static_exponential_base(true)
1222                .with_max_attempts(3)
1223                .with_retry_spec(RetrySpec::v2_1()),
1224        );
1225        let strategy = StandardRetryStrategy::new();
1226        cfg.interceptor_state().store_put(TokenBucket::default());
1227        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1228        let actual = strategy
1229            .should_attempt_retry(&ctx, &rc, &cfg)
1230            .expect("method is infallible for this use");
1231        // t_i = 50ms, retry_after = 10ms < t_i, so use t_i = 50ms
1232        assert_eq!(
1233            ShouldAttempt::YesAfterDelay(Duration::from_millis(50)),
1234            actual
1235        );
1236    }
1237
1238    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1239    #[test]
1240    fn v2_1_retry_after_above_t_i_plus_5s_capped() {
1241        let (mut cfg, rc, ctx) = setup_test(
1242            vec![RetryAction::retryable_error_with_explicit_delay(
1243                ErrorKind::ServerError,
1244                Duration::from_secs(10),
1245            )],
1246            RetryConfig::standard()
1247                .with_use_static_exponential_base(true)
1248                .with_max_attempts(3)
1249                .with_retry_spec(RetrySpec::v2_1()),
1250        );
1251        let strategy = StandardRetryStrategy::new();
1252        cfg.interceptor_state().store_put(TokenBucket::default());
1253        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1254        let actual = strategy
1255            .should_attempt_retry(&ctx, &rc, &cfg)
1256            .expect("method is infallible for this use");
1257        // t_i = 50ms, retry_after = 10s > t_i + 5s = 5.05s, so cap at 5.05s
1258        assert_eq!(
1259            ShouldAttempt::YesAfterDelay(Duration::from_millis(5050)),
1260            actual
1261        );
1262    }
1263
1264    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1265    #[test]
1266    fn v2_0_retry_after_capped_at_max_backoff() {
1267        let (mut cfg, rc, ctx) = setup_test(
1268            vec![RetryAction::retryable_error_with_explicit_delay(
1269                ErrorKind::ServerError,
1270                Duration::from_secs(30),
1271            )],
1272            RetryConfig::standard()
1273                .with_use_static_exponential_base(true)
1274                .with_max_attempts(3)
1275                .with_retry_spec(RetrySpec::v2_0()),
1276        );
1277        let strategy = StandardRetryStrategy::new();
1278        cfg.interceptor_state().store_put(TokenBucket::default());
1279        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1280        let actual = strategy
1281            .should_attempt_retry(&ctx, &rc, &cfg)
1282            .expect("method is infallible for this use");
1283        // v2.0: retry_after = 30s, capped at max_backoff = 20s
1284        assert_eq!(
1285            ShouldAttempt::YesAfterDelay(Duration::from_secs(20)),
1286            actual
1287        );
1288    }
1289
1290    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1291    #[test]
1292    fn long_polling_backs_off_when_token_bucket_empty() {
1293        let (mut cfg, rc, ctx) = setup_test(
1294            vec![RetryAction::server_error()],
1295            RetryConfig::standard()
1296                .with_use_static_exponential_base(true)
1297                .with_max_attempts(5)
1298                .with_retry_spec(RetrySpec::v2_1().with_long_polling(true)),
1299        );
1300        let strategy = StandardRetryStrategy::new();
1301        cfg.interceptor_state().store_put(TokenBucket::new(0));
1302        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1303        let hint = LongPollingBackoff::default();
1304        cfg.interceptor_state().store_put(hint.clone());
1305
1306        let result = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1307        assert_eq!(result, ShouldAttempt::No);
1308        assert_eq!(hint.take(), Some(Duration::from_millis(50)));
1309    }
1310
1311    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1312    #[test]
1313    fn non_long_polling_no_backoff_when_token_bucket_empty() {
1314        let (mut cfg, rc, ctx) = setup_test(
1315            vec![RetryAction::server_error()],
1316            RetryConfig::standard()
1317                .with_use_static_exponential_base(true)
1318                .with_max_attempts(5)
1319                .with_retry_spec(RetrySpec::v2_0()),
1320        );
1321        let strategy = StandardRetryStrategy::new();
1322        cfg.interceptor_state().store_put(TokenBucket::new(0));
1323        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1324
1325        let result = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1326        assert_eq!(result, ShouldAttempt::No);
1327    }
1328
1329    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1330    fn v2_1_token_bucket_with_capacity(capacity: usize) -> TokenBucket {
1331        TokenBucket::builder()
1332            .capacity(capacity)
1333            .retry_cost(DEFAULT_RETRY_COST)
1334            .throttling_retry_cost(THROTTLING_RETRY_COST)
1335            .timeout_retry_cost(DEFAULT_RETRY_TIMEOUT_COST)
1336            .build()
1337    }
1338
1339    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1340    fn v2_1_token_bucket() -> TokenBucket {
1341        v2_1_token_bucket_with_capacity(DEFAULT_CAPACITY)
1342    }
1343
1344    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1345    #[test]
1346    fn retry_eventually_succeeds() {
1347        let (mut cfg, rc, mut ctx) = setup_test(
1348            vec![RetryAction::server_error()],
1349            RetryConfig::standard()
1350                .with_use_static_exponential_base(true)
1351                .with_retry_spec(RetrySpec::v2_1()),
1352        );
1353        let strategy = StandardRetryStrategy::new();
1354        let tb = v2_1_token_bucket();
1355        cfg.interceptor_state().store_put(tb.clone());
1356
1357        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1358        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1359        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1360        assert_eq!(
1361            tb.available_permits(),
1362            DEFAULT_CAPACITY - DEFAULT_RETRY_COST as usize
1363        );
1364
1365        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1366        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1367        assert_eq!(should_retry.expect_delay(), Duration::from_millis(100));
1368        assert_eq!(
1369            tb.available_permits(),
1370            DEFAULT_CAPACITY - 2 * DEFAULT_RETRY_COST as usize
1371        );
1372
1373        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
1374        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1375        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1376        assert_eq!(no_retry, ShouldAttempt::No);
1377        assert_eq!(
1378            tb.available_permits(),
1379            DEFAULT_CAPACITY - DEFAULT_RETRY_COST as usize
1380        );
1381    }
1382
1383    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1384    #[test]
1385    fn fail_due_to_max_attempts_reached() {
1386        let (mut cfg, rc, ctx) = setup_test(
1387            vec![RetryAction::server_error()],
1388            RetryConfig::standard()
1389                .with_use_static_exponential_base(true)
1390                .with_retry_spec(RetrySpec::v2_1()),
1391        );
1392        let strategy = StandardRetryStrategy::new();
1393        let tb = v2_1_token_bucket();
1394        cfg.interceptor_state().store_put(tb.clone());
1395
1396        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1397        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1398        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1399        assert_eq!(
1400            tb.available_permits(),
1401            DEFAULT_CAPACITY - DEFAULT_RETRY_COST as usize
1402        );
1403
1404        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1405        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1406        assert_eq!(should_retry.expect_delay(), Duration::from_millis(100));
1407        assert_eq!(
1408            tb.available_permits(),
1409            DEFAULT_CAPACITY - 2 * DEFAULT_RETRY_COST as usize
1410        );
1411
1412        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1413        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1414        assert_eq!(no_retry, ShouldAttempt::No);
1415        assert_eq!(
1416            tb.available_permits(),
1417            DEFAULT_CAPACITY - 2 * DEFAULT_RETRY_COST as usize
1418        );
1419    }
1420
1421    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1422    #[test]
1423    fn retry_quota_reached_after_single_retry() {
1424        let (mut cfg, rc, ctx) = setup_test(
1425            vec![RetryAction::server_error()],
1426            RetryConfig::standard()
1427                .with_use_static_exponential_base(true)
1428                .with_max_attempts(5)
1429                .with_retry_spec(RetrySpec::v2_1()),
1430        );
1431        let strategy = StandardRetryStrategy::new();
1432        let tb = v2_1_token_bucket_with_capacity(DEFAULT_RETRY_COST as usize);
1433        cfg.interceptor_state().store_put(tb.clone());
1434
1435        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1436        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1437        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1438        assert_eq!(tb.available_permits(), 0);
1439
1440        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1441        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1442        assert_eq!(no_retry, ShouldAttempt::No);
1443        assert_eq!(tb.available_permits(), 0);
1444    }
1445
1446    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1447    #[test]
1448    fn no_retries_if_retry_quota_is_zero() {
1449        let (mut cfg, rc, ctx) = setup_test(
1450            vec![RetryAction::server_error()],
1451            RetryConfig::standard()
1452                .with_use_static_exponential_base(true)
1453                .with_retry_spec(RetrySpec::v2_1()),
1454        );
1455        let strategy = StandardRetryStrategy::new();
1456        let tb = v2_1_token_bucket_with_capacity(0);
1457        cfg.interceptor_state().store_put(tb.clone());
1458
1459        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1460        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1461        assert_eq!(no_retry, ShouldAttempt::No);
1462        assert_eq!(tb.available_permits(), 0);
1463    }
1464
1465    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1466    #[test]
1467    fn retry_stops_after_retry_quota_exhaustion() {
1468        let (mut cfg, rc, ctx) = setup_test(
1469            vec![RetryAction::server_error()],
1470            RetryConfig::standard()
1471                .with_use_static_exponential_base(true)
1472                .with_max_attempts(5)
1473                .with_retry_spec(RetrySpec::v2_1()),
1474        );
1475        let strategy = StandardRetryStrategy::new();
1476        let tb = v2_1_token_bucket_with_capacity(20);
1477        cfg.interceptor_state().store_put(tb.clone());
1478
1479        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1480        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1481        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1482        assert_eq!(tb.available_permits(), 20 - DEFAULT_RETRY_COST as usize);
1483
1484        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1485        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1486        assert_eq!(no_retry, ShouldAttempt::No);
1487        assert_eq!(tb.available_permits(), 20 - DEFAULT_RETRY_COST as usize);
1488    }
1489
1490    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1491    #[test]
1492    fn retry_quota_recovery_after_successful_responses() {
1493        let (mut cfg, rc, mut ctx) = setup_test(
1494            vec![RetryAction::server_error()],
1495            RetryConfig::standard()
1496                .with_use_static_exponential_base(true)
1497                .with_max_attempts(5)
1498                .with_retry_spec(RetrySpec::v2_1()),
1499        );
1500        let strategy = StandardRetryStrategy::new();
1501        let tb = v2_1_token_bucket_with_capacity(30);
1502        cfg.interceptor_state().store_put(tb.clone());
1503
1504        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1505        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1506        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1507        assert_eq!(tb.available_permits(), 30 - DEFAULT_RETRY_COST as usize);
1508
1509        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1510        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1511        assert_eq!(should_retry.expect_delay(), Duration::from_millis(100));
1512        assert_eq!(tb.available_permits(), 30 - 2 * DEFAULT_RETRY_COST as usize);
1513
1514        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
1515        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1516        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1517        assert_eq!(no_retry, ShouldAttempt::No);
1518        assert_eq!(tb.available_permits(), 30 - DEFAULT_RETRY_COST as usize);
1519
1520        ctx.set_output_or_error(Err(OrchestratorError::other("doesn't matter")));
1521        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1522        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1523        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1524        assert_eq!(tb.available_permits(), 30 - 2 * DEFAULT_RETRY_COST as usize);
1525
1526        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
1527        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1528        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1529        assert_eq!(no_retry, ShouldAttempt::No);
1530        assert_eq!(tb.available_permits(), 30 - DEFAULT_RETRY_COST as usize);
1531    }
1532
1533    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1534    #[test]
1535    fn throttling_error_token_bucket_drain_and_backoff() {
1536        let (mut cfg, rc, mut ctx) = setup_test(
1537            vec![RetryAction::retryable_error(ErrorKind::ThrottlingError)],
1538            RetryConfig::standard()
1539                .with_use_static_exponential_base(true)
1540                .with_retry_spec(RetrySpec::v2_1()),
1541        );
1542        let strategy = StandardRetryStrategy::new();
1543        let tb = v2_1_token_bucket();
1544        cfg.interceptor_state().store_put(tb.clone());
1545
1546        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1547        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1548        assert_eq!(should_retry.expect_delay(), Duration::from_secs(1));
1549        assert_eq!(
1550            tb.available_permits(),
1551            DEFAULT_CAPACITY - THROTTLING_RETRY_COST as usize
1552        );
1553
1554        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
1555        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1556        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1557        assert_eq!(no_retry, ShouldAttempt::No);
1558        assert_eq!(tb.available_permits(), DEFAULT_CAPACITY);
1559    }
1560
1561    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1562    #[test]
1563    fn long_polling_backoff_after_throttling_error_when_token_bucket_empty() {
1564        let (mut cfg, rc, ctx) = setup_test(
1565            vec![RetryAction::retryable_error(ErrorKind::ThrottlingError)],
1566            RetryConfig::standard()
1567                .with_use_static_exponential_base(true)
1568                .with_max_attempts(5)
1569                .with_retry_spec(RetrySpec::v2_1().with_long_polling(true)),
1570        );
1571        let strategy = StandardRetryStrategy::new();
1572        let tb = v2_1_token_bucket_with_capacity(0);
1573        cfg.interceptor_state().store_put(tb.clone());
1574        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1575        let hint = LongPollingBackoff::default();
1576        cfg.interceptor_state().store_put(hint.clone());
1577
1578        let result = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1579        assert_eq!(result, ShouldAttempt::No);
1580        assert_eq!(hint.take(), Some(Duration::from_secs(1)));
1581        assert_eq!(tb.available_permits(), 0);
1582    }
1583
1584    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1585    #[test]
1586    fn long_polling_max_attempts_exceeded_must_not_delay() {
1587        let (mut cfg, rc, ctx) = setup_test(
1588            vec![RetryAction::server_error()],
1589            RetryConfig::standard()
1590                .with_use_static_exponential_base(true)
1591                .with_max_attempts(2)
1592                .with_retry_spec(RetrySpec::v2_1().with_long_polling(true)),
1593        );
1594        let strategy = StandardRetryStrategy::new();
1595        let tb = v2_1_token_bucket();
1596        cfg.interceptor_state().store_put(tb.clone());
1597
1598        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1599        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1600        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1601
1602        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1603        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1604        assert_eq!(no_retry, ShouldAttempt::No);
1605    }
1606
1607    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1608    #[test]
1609    fn long_polling_success_must_not_delay() {
1610        let (mut cfg, rc, mut ctx) = setup_test(
1611            vec![RetryAction::server_error()],
1612            RetryConfig::standard()
1613                .with_use_static_exponential_base(true)
1614                .with_max_attempts(2)
1615                .with_retry_spec(RetrySpec::v2_1().with_long_polling(true)),
1616        );
1617        let strategy = StandardRetryStrategy::new();
1618        let tb = v2_1_token_bucket();
1619        cfg.interceptor_state().store_put(tb.clone());
1620
1621        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1622        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1623        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1624
1625        ctx.set_output_or_error(Ok(Output::doesnt_matter()));
1626        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1627        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1628        assert_eq!(no_retry, ShouldAttempt::No);
1629    }
1630
1631    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1632    #[test]
1633    fn long_polling_non_retryable_errors_must_not_delay() {
1634        let (mut cfg, rc, ctx) = setup_test(
1635            vec![RetryAction::NoActionIndicated],
1636            RetryConfig::standard()
1637                .with_use_static_exponential_base(true)
1638                .with_max_attempts(2)
1639                .with_retry_spec(RetrySpec::v2_1().with_long_polling(true)),
1640        );
1641        let strategy = StandardRetryStrategy::new();
1642        let tb = v2_1_token_bucket();
1643        cfg.interceptor_state().store_put(tb.clone());
1644        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1645
1646        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1647        assert_eq!(no_retry, ShouldAttempt::No);
1648    }
1649
1650    #[cfg(any(feature = "test-util", feature = "legacy-test-util"))]
1651    #[test]
1652    fn verify_max_backoff_time() {
1653        let (mut cfg, rc, ctx) = setup_test(
1654            vec![RetryAction::server_error()],
1655            RetryConfig::standard()
1656                .with_use_static_exponential_base(true)
1657                .with_max_attempts(5)
1658                .with_max_backoff(Duration::from_millis(200))
1659                .with_retry_spec(RetrySpec::v2_1()),
1660        );
1661        let strategy = StandardRetryStrategy::new();
1662        let tb = v2_1_token_bucket();
1663        cfg.interceptor_state().store_put(tb.clone());
1664
1665        cfg.interceptor_state().store_put(RequestAttempts::new(1));
1666        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1667        assert_eq!(should_retry.expect_delay(), Duration::from_millis(50));
1668        assert_eq!(
1669            tb.available_permits(),
1670            DEFAULT_CAPACITY - DEFAULT_RETRY_COST as usize
1671        );
1672
1673        cfg.interceptor_state().store_put(RequestAttempts::new(2));
1674        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1675        assert_eq!(should_retry.expect_delay(), Duration::from_millis(100));
1676        assert_eq!(
1677            tb.available_permits(),
1678            DEFAULT_CAPACITY - 2 * DEFAULT_RETRY_COST as usize
1679        );
1680
1681        cfg.interceptor_state().store_put(RequestAttempts::new(3));
1682        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1683        assert_eq!(should_retry.expect_delay(), Duration::from_millis(200));
1684        assert_eq!(
1685            tb.available_permits(),
1686            DEFAULT_CAPACITY - 3 * DEFAULT_RETRY_COST as usize
1687        );
1688
1689        // 50ms * 2^3 = 400ms, capped at max_backoff 200ms
1690        cfg.interceptor_state().store_put(RequestAttempts::new(4));
1691        let should_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1692        assert_eq!(should_retry.expect_delay(), Duration::from_millis(200));
1693        assert_eq!(
1694            tb.available_permits(),
1695            DEFAULT_CAPACITY - 4 * DEFAULT_RETRY_COST as usize
1696        );
1697
1698        cfg.interceptor_state().store_put(RequestAttempts::new(5));
1699        let no_retry = strategy.should_attempt_retry(&ctx, &rc, &cfg).unwrap();
1700        assert_eq!(no_retry, ShouldAttempt::No);
1701        assert_eq!(
1702            tb.available_permits(),
1703            DEFAULT_CAPACITY - 4 * DEFAULT_RETRY_COST as usize
1704        );
1705    }
1706}
aws_smithy_runtime/client/retries/strategy/standard.rs

aws_smithy_runtime/client/retries/strategy/
standard.rs
