AWS SDK

AWS SDK

rev. 1ed3d5371c342b375a8e2197cd2254b0dc08a51c (ignoring whitespace)

Files changed:

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/client/pool/registry.rs

@@ -797,797 +856,860 @@
  817    817   
        let registry = explicit_registry([partition(1)], ConnectionReuseScope::Pool, None).unwrap();
  818    818   
        let partition = explicit_partition(&registry, 1);
  819    819   
        let cell = registry
  820    820   
            .resolve_cell(&partition, &"https://example.com/".parse().unwrap())
  821    821   
            .unwrap();
  822    822   
  823    823   
        assert!(cell.admission().is_none());
  824    824   
        assert!(registry.admissions.lock().is_empty());
  825    825   
    }
  826    826   
         827  +
    /// Resolves the same first-use origin concurrently through one partition.
         828  +
    ///
         829  +
    /// Publication must retain one production `OriginCell` instance and return
         830  +
    /// that exact instance to both callers.
  827    831   
    #[test]
  828    832   
    fn first_cell_publication_is_stable_under_contention() {
  829    833   
        const THREADS: usize = 8;
  830    834   
        let registry = Arc::new(
  831    835   
            explicit_registry(
  832    836   
                [partition(1)],
  833    837   
                ConnectionReuseScope::default(),
  834    838   
                NonZeroUsize::new(1),
  835    839   
            )
  836    840   
            .unwrap(),
@@ -870,874 +934,944 @@
  890    894   
        let admission_policy =
  891    895   
            max_connections_per_host.map(|limit| AdmissionPolicy::new(limit, true));
  892    896   
        PartitionRegistry::new(
  893    897   
            Some(partitions.into_iter().collect()),
  894    898   
            reuse_scope,
  895    899   
            admission_policy,
  896    900   
            MaintenanceConfig::default(),
  897    901   
        )
  898    902   
    }
  899    903   
         904  +
    /// Resolves the same first-use origin through two concurrent production calls.
         905  +
    ///
         906  +
    /// Loom enumerates publication schedules while `PartitionRegistry` and
         907  +
    /// `PartitionState` perform their ordinary lookup and insertion transitions.
  900    908   
    #[test]
  901    909   
    fn first_cell_publication_is_stable_under_contention() {
  902    910   
        loom::model(|| {
  903    911   
            let registry = Arc::new(
  904    912   
                explicit_registry(
  905    913   
                    [Partition::new(PartitionId::from_index(1), TestSpawner)],
  906    914   
                    ConnectionReuseScope::default(),
  907    915   
                    None,
  908    916   
                )
  909    917   
                .unwrap(),
  910    918   
            );
  911    919   
            let partition = registry.partition(PartitionId::from_index(1)).unwrap();
  912    920   
  913    921   
            let first_registry = registry.clone();
  914    922   
            let first_partition = partition.clone();
  915    923   
            let first = loom::thread::spawn(move || {
  916    924   
                let uri: Uri = "https://example.com/".parse().unwrap();
  917    925   
                first_registry.resolve_cell(&first_partition, &uri).unwrap()
  918    926   
            });
  919    927   
            let second_registry = registry.clone();
  920    928   
            let second_partition = partition.clone();
  921    929   
            let second = loom::thread::spawn(move || {
  922    930   
                let uri: Uri = "https://example.com/".parse().unwrap();
  923    931   
                second_registry
  924    932   
                    .resolve_cell(&second_partition, &uri)
  925    933   
                    .unwrap()
  926    934   
            });
  927    935   
  928    936   
            let first = first.join().unwrap();
  929    937   
            let second = second.join().unwrap();
  930    938   
            assert!(Arc::ptr_eq(&first, &second));
  931    939   
            assert_eq!(1, partition.cell_count());
         940  +
            drop((first, second));
         941  +
            partition.maintenance.clear_modeled_cells_for_test();
  932    942   
        });
  933    943   
    }
  934    944   
}

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/hyper_legacy.rs

@@ -338,338 +404,401 @@
  358    358   
        let capture_connection = capture_connection(&mut request);
  359    359   
        if let Some(capture_smithy_connection) =
  360    360   
            request.extensions().get::<CaptureSmithyConnection>()
  361    361   
        {
  362    362   
            capture_smithy_connection
  363    363   
                .set_connection_retriever(move || extract_smithy_connection(&capture_connection));
  364    364   
        }
  365    365   
        let mut client = self.client.clone();
  366    366   
        let fut = client.call(request);
  367    367   
        HttpConnectorFuture::new(async move {
  368         -
            let response = fut
  369         -
                .await
  370         -
                .map_err(downcast_error)?
  371         -
                .map(SdkBody::from_body_0_4);
  372         -
            match HttpResponse::try_from(response) {
  373         -
                Ok(response) => Ok(response),
  374         -
                Err(err) => Err(ConnectorError::other(err.into(), None)),
         368  +
            match fut.await {
         369  +
                Ok(response) => HttpResponse::try_from(response.map(SdkBody::from_body_0_4))
         370  +
                    .map_err(|err| ConnectorError::other(err.into(), None)),
         371  +
                Err(err) => Err(downcast_error(err)),
  375    372   
            }
  376    373   
        })
  377    374   
    }
  378    375   
}
  379    376   
  380    377   
/// Downcast errors coming out of hyper into an appropriate `ConnectorError`
  381    378   
fn downcast_error(err: BoxError) -> ConnectorError {
  382    379   
    // is a `TimedOutError` (from aws_smithy_async::timeout) in the chain? if it is, this is a timeout
  383    380   
    if find_source::<TimedOutError>(err.as_ref()).is_some() {
  384    381   
        return ConnectorError::timeout(err);
@@ -464,461 +524,520 @@
  484    481   
        let key = CacheKey::from(settings);
  485    482   
        let mut connector = self.connector_cache.read().unwrap().get(&key).cloned();
  486    483   
        if connector.is_none() {
  487    484   
            let mut cache = self.connector_cache.write().unwrap();
  488    485   
            // Short-circuit if another thread already wrote a connector to the cache for this key
  489    486   
            if !cache.contains_key(&key) {
  490    487   
                let mut builder = HyperConnector::builder()
  491    488   
                    .hyper_builder(self.client_builder.clone())
  492    489   
                    .connector_settings(settings.clone());
  493    490   
                builder.set_sleep_impl(components.sleep_impl());
  494         -
  495    491   
                let start = components.time_source().map(|ts| ts.now());
  496    492   
                let tcp_connector = (self.tcp_connector_fn)();
  497    493   
                let end = components.time_source().map(|ts| ts.now());
  498    494   
                if let (Some(start), Some(end)) = (start, end) {
  499    495   
                    if let Ok(elapsed) = end.duration_since(start) {
  500    496   
                        tracing::debug!("new TCP connector created in {:?}", elapsed);
  501    497   
                    }
  502    498   
                }
  503    499   
                let connector = SharedHttpConnector::new(builder.build(tcp_connector));
  504    500   
                cache.insert(key.clone(), connector);

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/sync/loom.rs

@@ -1,1 +75,76 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
    6         -
//! Loom-backed synchronization primitives used by focused model tests.
           6  +
//! Loom-backed synchronization primitives used to run production pool state
           7  +
//! machines under enumerated thread schedules.
    7      8   
    8      9   
use ::std::ops::{Deref, DerefMut};
    9     10   
   10         -
pub(crate) use loom::sync::atomic::{AtomicBool, Ordering};
          11  +
pub(crate) use loom::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
   11     12   
pub(crate) use loom::sync::Arc;
   12     13   
   13     14   
loom::thread_local! {
   14     15   
    static HELD_POOL_LOCKS: ::std::cell::Cell<usize> = ::std::cell::Cell::new(0);
   15     16   
}
   16     17   
   17     18   
struct LockDepth;
   18     19   
   19     20   
impl LockDepth {
   20     21   
    fn enter() -> Self {
   21     22   
        HELD_POOL_LOCKS.with(|depth| {
   22     23   
            assert_eq!(
   23     24   
                0,
   24     25   
                depth.get(),
   25     26   
                "pool coordination locks must never be nested"
   26     27   
            );
   27     28   
            depth.set(1);
   28     29   
        });
   29     30   
        Self
   30     31   
    }
   31     32   
}
   32     33   
   33     34   
impl Drop for LockDepth {
   34     35   
    fn drop(&mut self) {
   35     36   
        HELD_POOL_LOCKS.with(|depth| {
   36     37   
            assert_eq!(1, depth.get(), "pool lock-depth tracking became unbalanced");
   37     38   
            depth.set(0);
   38     39   
        });
   39     40   
    }
   40     41   
}
   41     42   
   42     43   
/// Loom substitute for a non-owning registry reference.
   43     44   
///
   44     45   
/// Loom's modeled `Arc` has no weak-reference API. Keeping a strong reference
   45         -
/// in model tests preserves lookup behavior but cannot model target expiry.
          46  +
/// in Loom tests preserves lookup behavior but cannot model target expiry.
   46     47   
/// Ordinary tests cover the expired-target delivery fallback.
   47     48   
#[derive(Debug)]
   48     49   
pub(crate) struct Weak<T>(Arc<T>);
   49     50   
   50     51   
impl<T> Weak<T> {
   51     52   
    /// Creates the modeled registry reference.
   52     53   
    pub(crate) fn from_arc(value: &Arc<T>) -> Self {
   53     54   
        Self(value.clone())
   54     55   
    }
   55     56   

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/test_util/wire/connection.rs

@@ -76,76 +135,137 @@
   96     96   
/// Why a connection task stopped.
   97     97   
#[non_exhaustive]
   98     98   
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
   99     99   
pub enum ConnectionCloseReason {
  100    100   
    /// The peer closed the connection.
  101    101   
    ClientClosed,
  102    102   
    /// The script reached its end or explicitly closed the connection.
  103    103   
    ScriptCompleted,
  104    104   
    /// The script reset the connection.
  105    105   
    Reset,
         106  +
    /// The harness dropped the transport without a protocol close signal.
         107  +
    ScriptedTransportAbort,
  106    108   
    /// The harness was shut down.
  107    109   
    HarnessShutdown,
  108    110   
    /// The script failed.
  109    111   
    ScriptFailed,
  110    112   
}
  111    113   
  112    114   
/// An event recorded by the harness.
  113    115   
#[non_exhaustive]
  114    116   
#[derive(Clone, Debug, Eq, PartialEq)]
  115    117   
pub enum ConnectionEvent {

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/client.rs

@@ -1,1 +39,39 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
    6      6   
//! Client configuration and request helpers shared by connection behavior tests.
    7      7   
    8      8   
use aws_smithy_async::rt::sleep::{SharedAsyncSleep, TokioSleep};
    9         -
use aws_smithy_async::time::SystemTimeSource;
           9  +
use aws_smithy_async::time::{SystemTimeSource, TimeSource};
   10     10   
use aws_smithy_http_client::pool::{Client as PoolClient, ConnectionPool};
   11     11   
use aws_smithy_http_client::proxy::ProxyConfig;
   12     12   
#[cfg(any(feature = "__rustls", feature = "s2n-tls"))]
   13     13   
use aws_smithy_http_client::tls;
   14     14   
use aws_smithy_http_client::{Builder, Connector};
   15     15   
use aws_smithy_runtime_api::client::dns::SharedDnsResolver;
   16     16   
use aws_smithy_runtime_api::client::http::{
   17     17   
    http_client_fn, HttpClient, HttpConnector, HttpConnectorSettings, SharedHttpClient,
   18     18   
    SharedHttpConnector,
   19     19   
};
@@ -230,230 +289,303 @@
  250    250   
    client: &SharedHttpClient,
  251    251   
    settings: HttpConnectorSettings,
  252    252   
) -> SharedHttpConnector {
  253    253   
    client.http_connector(&settings, &runtime_components())
  254    254   
}
  255    255   
  256    256   
pub(crate) fn connector(client: &SharedHttpClient) -> SharedHttpConnector {
  257    257   
    connector_with_settings(client, HttpConnectorSettings::builder().build())
  258    258   
}
  259    259   
         260  +
/// Builds a connector whose request telemetry uses `time_source`.
         261  +
#[allow(dead_code)]
         262  +
pub(crate) fn connector_with_time_source(
         263  +
    client: &SharedHttpClient,
         264  +
    time_source: impl TimeSource + 'static,
         265  +
) -> SharedHttpConnector {
         266  +
    let components = RuntimeComponentsBuilder::for_tests()
         267  +
        .with_time_source(Some(time_source))
         268  +
        .with_sleep_impl(Some(SharedAsyncSleep::new(TokioSleep::new())))
         269  +
        .build()
         270  +
        .expect("valid runtime components");
         271  +
    client.http_connector(&HttpConnectorSettings::builder().build(), &components)
         272  +
}
         273  +
  260    274   
pub(crate) async fn send_request(
  261    275   
    connector: &SharedHttpConnector,
  262    276   
    request: HttpRequest,
  263    277   
) -> Result<HttpResponse, ConnectorError> {
  264    278   
    tokio::time::timeout(WAIT, connector.call(request))
  265    279   
        .await
  266    280   
        .expect("request should finish within the outer deadline")
  267    281   
}
  268    282   
  269    283   
pub(crate) async fn send_and_collect(

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/h2.rs

@@ -171,171 +293,323 @@
  191    191   
  192    192   
    pub(crate) fn body(mut self, body: H2BodyPlan) -> Self {
  193    193   
        self.body = body;
  194    194   
        self
  195    195   
    }
  196    196   
}
  197    197   
  198    198   
#[derive(Clone, Debug)]
  199    199   
pub(crate) enum H2StreamScript {
  200    200   
    Respond(H2Response),
         201  +
    /// Waits at the response-head boundary before sending the response.
         202  +
    RespondAfter {
         203  +
        response: H2Response,
         204  +
        send: GateWaiter,
         205  +
    },
  201    206   
    /// Sends the response before a gate permits the request body to be drained.
  202    207   
    RespondBeforeReceivingRequestBody {
  203    208   
        response: H2Response,
  204    209   
        receive: GateWaiter,
  205    210   
    },
  206    211   
    Reset(Reason),
  207    212   
}
  208    213   
  209    214   
impl H2StreamScript {
  210    215   
    pub(crate) fn respond(response: H2Response) -> Self {
  211    216   
        Self::Respond(response)
  212    217   
    }
  213    218   
         219  +
    /// Delays the response head until the gate is released.
         220  +
    pub(crate) fn respond_after(response: H2Response, send: GateWaiter) -> Self {
         221  +
        Self::RespondAfter { response, send }
         222  +
    }
         223  +
  214    224   
    /// Delays request-body reads until after the response and gate release.
  215    225   
    pub(crate) fn respond_before_receiving_request_body(
  216    226   
        response: H2Response,
  217    227   
        receive: GateWaiter,
  218    228   
    ) -> Self {
  219    229   
        Self::RespondBeforeReceivingRequestBody { response, receive }
  220    230   
    }
  221    231   
  222    232   
    pub(crate) fn reset(reason: Reason) -> Self {
  223    233   
        Self::Reset(reason)
  224    234   
    }
  225    235   
}
  226    236   
  227    237   
#[derive(Clone, Debug, Default)]
  228    238   
pub(crate) struct H2ConnectionScript {
  229    239   
    routes: HashMap<String, H2StreamScript>,
  230    240   
    fallback: Option<H2StreamScript>,
  231    241   
    allow_handshake_abandonment: bool,
  232    242   
    goaway_on_ready: bool,
         243  +
    client_close_behavior: ClientCloseBehavior,
         244  +
}
         245  +
         246  +
/// How a scripted connection handles active stream tasks after client close.
         247  +
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
         248  +
enum ClientCloseBehavior {
         249  +
    /// Wait for stream scripts and report their failures.
         250  +
    #[default]
         251  +
    Drain,
         252  +
    /// Abort scripts that a test deliberately leaves blocked.
         253  +
    AbortStreams,
  233    254   
}
  234    255   
  235    256   
impl H2ConnectionScript {
  236    257   
    pub(crate) fn new() -> Self {
  237    258   
        Self::default()
  238    259   
    }
  239    260   
  240    261   
    pub(crate) fn route(mut self, path: impl Into<String>, script: H2StreamScript) -> Self {
  241    262   
        let path = path.into();
  242    263   
        assert!(
  243    264   
            self.routes.insert(path.clone(), script).is_none(),
  244    265   
            "duplicate H2 route {path:?}"
  245    266   
        );
  246    267   
        self
  247    268   
    }
  248    269   
  249    270   
    pub(crate) fn fallback(mut self, script: H2StreamScript) -> Self {
  250    271   
        self.fallback = Some(script);
  251    272   
        self
  252    273   
    }
  253    274   
  254    275   
    pub(crate) fn allow_handshake_abandonment(mut self) -> Self {
  255    276   
        self.allow_handshake_abandonment = true;
  256    277   
        self
  257    278   
    }
  258    279   
  259    280   
    pub(crate) fn goaway_on_ready(mut self) -> Self {
  260    281   
        self.goaway_on_ready = true;
  261    282   
        self
  262    283   
    }
  263    284   
         285  +
    /// Aborts active stream scripts when the client closes the connection.
         286  +
    ///
         287  +
    /// Use this only when the test intentionally strands a stream while
         288  +
    /// terminating the runtime that owns its transport.
         289  +
    pub(crate) fn abort_streams_on_client_close(mut self) -> Self {
         290  +
        self.client_close_behavior = ClientCloseBehavior::AbortStreams;
         291  +
        self
         292  +
    }
         293  +
  264    294   
    fn script_for(&self, path: &str) -> Option<H2StreamScript> {
  265    295   
        self.routes.get(path).or(self.fallback.as_ref()).cloned()
  266    296   
    }
  267    297   
}
  268    298   
  269    299   
#[derive(Clone, Debug)]
  270    300   
pub(crate) enum H2ConnectionPlan {
  271    301   
    Queue(VecDeque<H2ConnectionScript>),
  272    302   
    Unbounded(H2ConnectionScript),
  273    303   
}
@@ -368,398 +427,458 @@
  388    418   
            .lock()
  389    419   
            .unwrap_or_else(|err| err.into_inner())
  390    420   
            .get(&connection_id)
  391    421   
            .cloned()
  392    422   
    }
  393    423   
}
  394    424   
  395    425   
#[derive(Debug)]
  396    426   
enum ConnectionCommand {
  397    427   
    GracefulShutdown(oneshot::Sender<()>),
         428  +
    AbortTransport(oneshot::Sender<()>),
  398    429   
}
  399    430   
  400    431   
#[derive(Debug, Default)]
  401    432   
pub(crate) struct H2ServerBuilder {
  402    433   
    connections: Option<H2ConnectionPlan>,
  403    434   
}
  404    435   
  405    436   
impl H2ServerBuilder {
  406    437   
    pub(crate) fn connections(mut self, connections: H2ConnectionPlan) -> Self {
  407    438   
        self.connections = Some(connections);
@@ -507,538 +566,625 @@
  527    558   
                    "timed out waiting for H2 connection {connection_id:?} to start graceful shutdown"
  528    559   
                ))
  529    560   
            })?
  530    561   
            .map_err(|_| {
  531    562   
                H2HarnessError::new(format!(
  532    563   
                    "H2 connection {connection_id:?} closed before graceful shutdown started"
  533    564   
                ))
  534    565   
            })
  535    566   
    }
  536    567   
         568  +
    /// Drops one active transport without sending GOAWAY.
         569  +
    pub(crate) async fn abort_transport(
         570  +
        &self,
         571  +
        connection_id: H2ConnectionId,
         572  +
    ) -> Result<(), H2HarnessError> {
         573  +
        let control = self.state.control(connection_id).ok_or_else(|| {
         574  +
            H2HarnessError::new(format!("H2 connection {connection_id:?} is not active"))
         575  +
        })?;
         576  +
        let (acknowledged, ack) = oneshot::channel();
         577  +
        control
         578  +
            .send(ConnectionCommand::AbortTransport(acknowledged))
         579  +
            .map_err(|_| {
         580  +
                H2HarnessError::new(format!("failed to abort H2 transport {connection_id:?}"))
         581  +
            })?;
         582  +
        tokio::time::timeout(WAIT, ack)
         583  +
            .await
         584  +
            .map_err(|_| {
         585  +
                H2HarnessError::new(format!(
         586  +
                    "timed out waiting to abort H2 transport {connection_id:?}"
         587  +
                ))
         588  +
            })?
         589  +
            .map_err(|_| {
         590  +
                H2HarnessError::new(format!(
         591  +
                    "H2 connection {connection_id:?} closed before its transport was aborted"
         592  +
                ))
         593  +
            })
         594  +
    }
         595  +
  537    596   
    pub(crate) async fn shutdown(mut self) -> Result<(), H2HarnessError> {
  538    597   
        self.shutdown.send_replace(true);
  539    598   
        let mut listener_task = self
  540    599   
            .listener_task
  541    600   
            .take()
  542    601   
            .expect("listener task should exist until shutdown");
  543    602   
        match tokio::time::timeout(WAIT, &mut listener_task).await {
  544    603   
            Ok(Ok(())) => {}
  545    604   
            Ok(Err(err)) => {
  546    605   
                return Err(H2HarnessError::new(format!(
@@ -711,770 +791,856 @@
  731    790   
            }
  732    791   
            Ok(Err(err)) => return Err(H2HarnessError::new(format!("H2 handshake failed: {err}"))),
  733    792   
        };
  734    793   
    state.record_event(H2Event::H2Ready { connection_id });
  735    794   
    if script.goaway_on_ready {
  736    795   
        connection.abrupt_shutdown(Reason::NO_ERROR);
  737    796   
    }
  738    797   
  739    798   
    let mut stream_tasks = JoinSet::new();
  740    799   
    let mut shutting_down = false;
         800  +
    let mut transport_aborted = false;
  741    801   
    let mut graceful_shutdown = script.goaway_on_ready;
  742    802   
    let mut control_open = true;
  743    803   
    // Tracks whether the connection ended because the client closed it (accept returned None)
  744    804   
    // vs. a scripted GOAWAY completing.
  745    805   
    let mut client_initiated_close = false;
  746    806   
  747    807   
    loop {
  748    808   
        tokio::select! {
  749    809   
            changed = shutdown.changed(), if !shutting_down => {
  750    810   
                if changed.is_err() || *shutdown.borrow() {
  751    811   
                    shutting_down = true;
  752    812   
                    connection.abrupt_shutdown(Reason::NO_ERROR);
  753    813   
                }
  754    814   
            }
  755    815   
            command = control.recv(), if control_open => {
  756    816   
                match command {
  757    817   
                    Some(ConnectionCommand::GracefulShutdown(acknowledged)) => {
  758    818   
                        graceful_shutdown = true;
  759    819   
                        connection.graceful_shutdown();
  760    820   
                        let _ = acknowledged.send(());
  761    821   
                    }
         822  +
                    Some(ConnectionCommand::AbortTransport(acknowledged)) => {
         823  +
                        transport_aborted = true;
         824  +
                        let _ = acknowledged.send(());
         825  +
                        break;
         826  +
                    }
  762    827   
                    None => control_open = false,
  763    828   
                }
  764    829   
            }
  765    830   
            accepted = connection.accept() => {
  766    831   
                match accepted {
  767    832   
                    Some(Ok((request, mut respond))) => {
  768    833   
                        let stream_id = respond.stream_id().as_u32();
  769    834   
                        let method = request.method().clone();
  770    835   
                        let path = request.uri().path().to_string();
  771    836   
                        let proxy_authorization = request
@@ -808,873 +923,1025 @@
  828    893   
                        break;
  829    894   
                    }
  830    895   
                }
  831    896   
            }
  832    897   
            completed = stream_tasks.join_next(), if !stream_tasks.is_empty() => {
  833    898   
                record_stream_task_result(&state, completed);
  834    899   
            }
  835    900   
        }
  836    901   
    }
  837    902   
  838         -
    if shutting_down {
         903  +
    let abort_streams = shutting_down
         904  +
        || transport_aborted
         905  +
        || (client_initiated_close
         906  +
            && script.client_close_behavior == ClientCloseBehavior::AbortStreams);
         907  +
    if abort_streams {
  839    908   
        stream_tasks.abort_all();
  840    909   
        while stream_tasks.join_next().await.is_some() {}
  841         -
        Ok(ConnectionCloseReason::HarnessShutdown)
         910  +
        Ok(aborted_connection_close_reason(
         911  +
            transport_aborted,
         912  +
            shutting_down,
         913  +
            client_initiated_close,
         914  +
        ))
  842    915   
    } else {
  843    916   
        let drain = async {
  844    917   
            while let Some(completed) = stream_tasks.join_next().await {
  845    918   
                record_stream_task_result(&state, Some(completed));
  846    919   
            }
  847    920   
        };
  848    921   
        if tokio::time::timeout(WAIT, drain).await.is_err() {
  849    922   
            state.record_failure(H2HarnessError::new(format!(
  850    923   
                "H2 connection {connection_id:?} closed before its stream scripts completed"
  851    924   
            )));
  852    925   
            stream_tasks.abort_all();
  853    926   
            while stream_tasks.join_next().await.is_some() {}
  854    927   
        }
  855    928   
        if client_initiated_close {
  856    929   
            Ok(ConnectionCloseReason::ClientClosed)
  857    930   
        } else {
  858    931   
            Ok(ConnectionCloseReason::ScriptCompleted)
  859    932   
        }
  860    933   
    }
  861    934   
}
  862    935   
         936  +
fn aborted_connection_close_reason(
         937  +
    transport_aborted: bool,
         938  +
    shutting_down: bool,
         939  +
    client_initiated_close: bool,
         940  +
) -> ConnectionCloseReason {
         941  +
    debug_assert!(transport_aborted || shutting_down || client_initiated_close);
         942  +
    if transport_aborted {
         943  +
        ConnectionCloseReason::ScriptedTransportAbort
         944  +
    } else if shutting_down {
         945  +
        ConnectionCloseReason::HarnessShutdown
         946  +
    } else {
         947  +
        ConnectionCloseReason::ClientClosed
         948  +
    }
         949  +
}
         950  +
         951  +
#[test]
         952  +
fn harness_shutdown_precedes_client_close() {
         953  +
    assert_eq!(
         954  +
        ConnectionCloseReason::HarnessShutdown,
         955  +
        aborted_connection_close_reason(false, true, true,)
         956  +
    );
         957  +
}
         958  +
  863    959   
fn record_stream_task_result(
  864    960   
    state: &SharedState,
  865    961   
    completed: Option<Result<Result<(), H2HarnessError>, tokio::task::JoinError>>,
  866    962   
) {
  867    963   
    match completed {
  868    964   
        Some(Ok(Ok(()))) | None => {}
  869    965   
        Some(Ok(Err(err))) => state.record_failure(err),
  870    966   
        Some(Err(err)) => {
  871    967   
            state.record_failure(H2HarnessError::new(format!("H2 stream task failed: {err}")))
  872    968   
        }
  873    969   
    }
  874    970   
}
  875    971   
  876    972   
async fn run_stream(
  877    973   
    connection_id: H2ConnectionId,
  878    974   
    stream_id: u32,
  879    975   
    script: H2StreamScript,
  880    976   
    mut respond: SendResponse<Bytes>,
  881    977   
    state: Arc<SharedState>,
  882    978   
    held_request: Option<(RecvStream, GateWaiter)>,
  883    979   
) -> Result<(), H2HarnessError> {
         980  +
    if let H2StreamScript::RespondAfter { send, .. } = &script {
         981  +
        send.wait()
         982  +
            .await
         983  +
            .map_err(|err| H2HarnessError::new(format!("H2 response-head gate failed: {err}")))?;
         984  +
    }
  884    985   
    match script {
  885    986   
        H2StreamScript::Reset(reason) => {
  886    987   
            respond.send_reset(reason);
  887    988   
            state.record_event(H2Event::ResetSent {
  888    989   
                connection_id,
  889    990   
                stream_id,
  890    991   
                reason,
  891    992   
            });
  892    993   
        }
  893    994   
        H2StreamScript::Respond(response)
         995  +
        | H2StreamScript::RespondAfter { response, send: _ }
  894    996   
        | H2StreamScript::RespondBeforeReceivingRequestBody {
  895    997   
            response,
  896    998   
            receive: _,
  897    999   
        } => {
  898   1000   
            let end_stream =
  899   1001   
                matches!(&response.body, H2BodyPlan::Complete(body) if body.is_empty());
  900   1002   
            let response_head = Response::builder()
  901   1003   
                .status(response.status)
  902   1004   
                .body(())
  903   1005   
                .map_err(|err| {

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/runtime.rs

@@ -0,1 +0,122 @@
           1  +
/*
           2  +
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
           3  +
 * SPDX-License-Identifier: Apache-2.0
           4  +
 */
           5  +
           6  +
//! Independently driven Tokio runtimes for partition-placement tests.
           7  +
           8  +
use aws_smithy_http_client::pool::DriverSpawner;
           9  +
use std::future::Future;
          10  +
use std::pin::Pin;
          11  +
use std::sync::atomic::{AtomicUsize, Ordering};
          12  +
use std::sync::{mpsc, Arc};
          13  +
use std::thread::{self, JoinHandle};
          14  +
use tokio::runtime::{Handle, Id};
          15  +
use tokio::sync::oneshot;
          16  +
          17  +
/// A current-thread Tokio runtime driven by its own operating-system thread.
          18  +
#[derive(Debug)]
          19  +
pub(crate) struct DrivenRuntime {
          20  +
    handle: Handle,
          21  +
    runtime_id: Id,
          22  +
    shutdown: Option<oneshot::Sender<()>>,
          23  +
    thread: Option<JoinHandle<()>>,
          24  +
    submitted_tasks: Arc<AtomicUsize>,
          25  +
}
          26  +
          27  +
impl DrivenRuntime {
          28  +
    /// Starts a current-thread runtime and waits until its handle is available.
          29  +
    pub(crate) fn start(name: &str) -> Self {
          30  +
        let (ready_tx, ready_rx) = mpsc::sync_channel(1);
          31  +
        let (shutdown, shutdown_rx) = oneshot::channel();
          32  +
        let submitted_tasks = Arc::new(AtomicUsize::new(0));
          33  +
        let thread = thread::Builder::new()
          34  +
            .name(name.to_owned())
          35  +
            .spawn(move || {
          36  +
                let runtime = tokio::runtime::Builder::new_current_thread()
          37  +
                    .enable_all()
          38  +
                    .build()
          39  +
                    .expect("test runtime should build");
          40  +
                let handle = runtime.handle().clone();
          41  +
                let runtime_id = handle.id();
          42  +
                ready_tx
          43  +
                    .send((handle, runtime_id))
          44  +
                    .expect("runtime owner should receive its handle");
          45  +
                runtime.block_on(async {
          46  +
                    let _ = shutdown_rx.await;
          47  +
                });
          48  +
            })
          49  +
            .expect("test runtime thread should start");
          50  +
        let (handle, runtime_id) = ready_rx
          51  +
            .recv()
          52  +
            .expect("test runtime should publish its handle");
          53  +
        Self {
          54  +
            handle,
          55  +
            runtime_id,
          56  +
            shutdown: Some(shutdown),
          57  +
            thread: Some(thread),
          58  +
            submitted_tasks,
          59  +
        }
          60  +
    }
          61  +
          62  +
    /// Returns a spawner that records task placement on this runtime.
          63  +
    pub(crate) fn driver_spawner(&self) -> RuntimeDriverSpawner {
          64  +
        RuntimeDriverSpawner {
          65  +
            handle: self.handle.clone(),
          66  +
            submitted_tasks: self.submitted_tasks.clone(),
          67  +
        }
          68  +
    }
          69  +
          70  +
    /// Spawns test orchestration work on this runtime.
          71  +
    pub(crate) fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>
          72  +
    where
          73  +
        F: Future + Send + 'static,
          74  +
        F::Output: Send + 'static,
          75  +
    {
          76  +
        self.handle.spawn(future)
          77  +
    }
          78  +
          79  +
    /// Returns the runtime's stable Tokio identity.
          80  +
    pub(crate) fn id(&self) -> Id {
          81  +
        self.runtime_id
          82  +
    }
          83  +
          84  +
    /// Returns how many connection-owned tasks were submitted to this runtime.
          85  +
    pub(crate) fn submitted_tasks(&self) -> usize {
          86  +
        self.submitted_tasks.load(Ordering::SeqCst)
          87  +
    }
          88  +
          89  +
    /// Stops the runtime, drops all runtime-owned tasks, and joins its thread.
          90  +
    pub(crate) fn shutdown(mut self) {
          91  +
        self.stop();
          92  +
    }
          93  +
          94  +
    fn stop(&mut self) {
          95  +
        if let Some(shutdown) = self.shutdown.take() {
          96  +
            let _ = shutdown.send(());
          97  +
        }
          98  +
        if let Some(thread) = self.thread.take() {
          99  +
            thread.join().expect("test runtime thread should not panic");
         100  +
        }
         101  +
    }
         102  +
}
         103  +
         104  +
impl Drop for DrivenRuntime {
         105  +
    fn drop(&mut self) {
         106  +
        self.stop();
         107  +
    }
         108  +
}
         109  +
         110  +
/// Places connection-owned tasks on one [`DrivenRuntime`].
         111  +
#[derive(Clone, Debug)]
         112  +
pub(crate) struct RuntimeDriverSpawner {
         113  +
    handle: Handle,
         114  +
    submitted_tasks: Arc<AtomicUsize>,
         115  +
}
         116  +
         117  +
impl DriverSpawner for RuntimeDriverSpawner {
         118  +
    fn spawn(&self, driver: Pin<Box<dyn Future<Output = ()> + Send + 'static>>) {
         119  +
        self.submitted_tasks.fetch_add(1, Ordering::SeqCst);
         120  +
        drop(self.handle.spawn(driver));
         121  +
    }
         122  +
}

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/connection_pool_test.rs

@@ -1,1 +66,81 @@
    9      9   
    feature = "wire-mock",
   10     10   
    feature = "default-client",
   11     11   
    feature = "rt-tokio"
   12     12   
))]
   13     13   
   14     14   
mod common {
   15     15   
    #[allow(dead_code)]
   16     16   
    pub(crate) mod client;
   17     17   
}
   18     18   
          19  +
#[path = "common/runtime.rs"]
          20  +
mod runtime;
          21  +
          22  +
use aws_smithy_async::test_util::ManualTimeSource;
   19     23   
use aws_smithy_http_client::pool::{
   20     24   
    Client, ConnectPath, ConnectionEstablishmentId, ConnectionEstablishmentStage, ConnectionEvent,
   21     25   
    ConnectionPool, ConnectionProtocol, ConnectionReuseScope, OriginKey, Partition, PartitionId,
   22     26   
    TokioDriverSpawner,
   23     27   
};
   24     28   
use aws_smithy_http_client::test_util::wire::connection::{
   25     29   
    BodyPlan, ConnectionCloseReason, ConnectionEvent as WireConnectionEvent, ConnectionTestHarness,
   26         -
    EndpointPlan, Http1Response, Http1Script, ManualGate,
          30  +
    EndpointPlan, Http1Response, Http1Script, ManualGate, SocketScript,
          31  +
};
          32  +
use aws_smithy_runtime_api::client::connection::{CaptureSmithyConnection, ConnectionId};
          33  +
use aws_smithy_runtime_api::client::http::telemetry::{
          34  +
    CaptureHttpAttemptTelemetry, ConnectionUsage,
   27     35   
};
   28         -
use aws_smithy_runtime_api::client::connection::ConnectionId;
   29     36   
use aws_smithy_runtime_api::client::http::{SharedHttpClient, SharedHttpConnector};
   30     37   
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
          38  +
use aws_smithy_types::body::SdkBody;
          39  +
use bytes::Bytes;
   31     40   
use common::client as test_client;
          41  +
use http_body_1x::{Body, Frame, SizeHint};
          42  +
use std::convert::Infallible;
          43  +
use std::future::Future;
   32     44   
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
          45  +
use std::pin::Pin;
   33     46   
use std::sync::{Arc, Mutex};
   34         -
use std::time::Duration;
          47  +
use std::task::{Context, Poll};
          48  +
use std::time::{Duration, UNIX_EPOCH};
   35     49   
use tokio::io::{AsyncReadExt, AsyncWriteExt};
   36     50   
use tokio::net::TcpListener;
          51  +
use tokio::sync::oneshot;
   37     52   
   38     53   
const IP1: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
   39     54   
const IP2: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2));
   40     55   
   41     56   
#[derive(Debug)]
   42     57   
enum ObservedConnectionEvent {
   43     58   
    Failed {
   44     59   
        establishment: ConnectionEstablishmentId,
   45     60   
        origin: OriginKey,
   46     61   
        stage: ConnectionEstablishmentStage,
@@ -292,307 +407,590 @@
  312    327   
    let (status, body) = test_client::get_and_collect(&connector, &url).await;
  313    328   
    assert_eq!((status, body.as_slice()), (200, b"custom dns".as_slice()));
  314    329   
    assert_eq!(1, harness.dns_lookup_count());
  315    330   
  316    331   
    drop(connector);
  317    332   
    drop(client);
  318    333   
    drop(pool);
  319    334   
    harness.shutdown().await.expect("clean harness shutdown");
  320    335   
}
  321    336   
         337  +
#[tokio::test]
         338  +
async fn h1_attempt_telemetry_distinguishes_fresh_and_reused_connections() {
         339  +
    let harness = ConnectionTestHarness::builder()
         340  +
        .endpoint(
         341  +
            IP1,
         342  +
            Http1Script::responses([
         343  +
                Http1Response::ok().body("first"),
         344  +
                Http1Response::ok().body("second"),
         345  +
            ]),
         346  +
        )
         347  +
        .build()
         348  +
        .await
         349  +
        .expect("harness should start");
         350  +
    let opened = Arc::new(Mutex::new(None));
         351  +
    let pool = ConnectionPool::builder()
         352  +
        .event_listener({
         353  +
            let opened = opened.clone();
         354  +
            move |event: &ConnectionEvent<'_>| {
         355  +
                if let ConnectionEvent::Opened(event) = event {
         356  +
                    *opened.lock().unwrap() =
         357  +
                        Some((event.connection().id(), event.stats().total_duration()));
         358  +
                }
         359  +
            }
         360  +
        })
         361  +
        .build_http()
         362  +
        .expect("valid pool");
         363  +
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
         364  +
    let connector = connector(&client);
         365  +
         366  +
    let first_connection_capture = CaptureSmithyConnection::new();
         367  +
    let first_capture = CaptureHttpAttemptTelemetry::new();
         368  +
    let mut first = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         369  +
    first.add_extension(first_connection_capture.clone());
         370  +
    first.add_extension(first_capture.clone());
         371  +
    test_client::send_and_collect(&connector, first).await;
         372  +
         373  +
    let second_capture = CaptureHttpAttemptTelemetry::new();
         374  +
    let mut second = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         375  +
    second.add_extension(second_capture.clone());
         376  +
    test_client::send_and_collect(&connector, second).await;
         377  +
         378  +
    let first = first_capture.get();
         379  +
    let second = second_capture.get();
         380  +
    assert!(first.connector_call_duration().is_some());
         381  +
    assert!(second.connector_call_duration().is_some());
         382  +
    assert_eq!(
         383  +
        first.acquisition().expect("first acquisition").usage(),
         384  +
        ConnectionUsage::Fresh
         385  +
    );
         386  +
    assert_eq!(
         387  +
        second.acquisition().expect("second acquisition").usage(),
         388  +
        ConnectionUsage::Reused
         389  +
    );
         390  +
    let first_connection = first.connection().expect("first connection metadata");
         391  +
    let second_connection = second.connection().expect("second connection metadata");
         392  +
    let captured_connection = first_connection_capture
         393  +
        .get()
         394  +
        .expect("connection capture should contain selected metadata");
         395  +
    assert_eq!(
         396  +
        captured_connection.connection_id(),
         397  +
        first_connection.connection_id()
         398  +
    );
         399  +
    assert_eq!(
         400  +
        captured_connection.establishment(),
         401  +
        first_connection.establishment(),
         402  +
        "connection and attempt captures must observe the same selected connection"
         403  +
    );
         404  +
    assert_eq!(
         405  +
        first_connection.connection_id(),
         406  +
        second_connection.connection_id()
         407  +
    );
         408  +
    assert_eq!(
         409  +
        first_connection.establishment(),
         410  +
        second_connection.establishment(),
         411  +
        "reuse must retain the establishment that created the connection"
         412  +
    );
         413  +
    let first_establishment = first_connection
         414  +
        .establishment()
         415  +
        .expect("first connection establishment");
         416  +
    let (opened_connection, opened_duration) =
         417  +
        opened.lock().unwrap().expect("opened connection event");
         418  +
    assert_eq!(
         419  +
        first_connection.connection_id(),
         420  +
        Some(opened_connection),
         421  +
        "request telemetry and lifecycle events must identify the same connection"
         422  +
    );
         423  +
    assert_eq!(
         424  +
        first_establishment.total_duration(),
         425  +
        opened_duration,
         426  +
        "request metadata and the opened event must share one establishment measurement"
         427  +
    );
         428  +
    assert_eq!(1, harness.tcp_accepted_count());
         429  +
         430  +
    drop(connector);
         431  +
    drop(client);
         432  +
    drop(pool);
         433  +
    harness.shutdown().await.expect("clean harness shutdown");
         434  +
}
         435  +
         436  +
#[tokio::test]
         437  +
async fn acquisition_duration_ends_when_hyper_accepts_the_request() {
         438  +
    let response_gate = ManualGate::new();
         439  +
    let harness = ConnectionTestHarness::builder()
         440  +
        .endpoint(
         441  +
            IP1,
         442  +
            SocketScript::new()
         443  +
                .read_http1_request()
         444  +
                .wait(response_gate.waiter())
         445  +
                .write_all(
         446  +
                    "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: keep-alive\r\n\r\nok",
         447  +
                )
         448  +
                .await_client_close(),
         449  +
        )
         450  +
        .build()
         451  +
        .await
         452  +
        .expect("harness should start");
         453  +
    let pool_time = ManualTimeSource::new(UNIX_EPOCH);
         454  +
    let attempt_time = ManualTimeSource::new(UNIX_EPOCH);
         455  +
    let pool = ConnectionPool::builder()
         456  +
        .time_source(pool_time)
         457  +
        .build_http()
         458  +
        .expect("valid pool");
         459  +
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
         460  +
    let connector = test_client::connector_with_time_source(&client, attempt_time.clone());
         461  +
    let capture = CaptureHttpAttemptTelemetry::new();
         462  +
    let mut request = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         463  +
    request.add_extension(capture.clone());
         464  +
         465  +
    let send = tokio::spawn({
         466  +
        let connector = connector.clone();
         467  +
        async move { test_client::send_and_collect(&connector, request).await }
         468  +
    });
         469  +
    response_gate
         470  +
        .wait_until_reached(test_client::WAIT)
         471  +
        .await
         472  +
        .expect("server should receive the accepted request");
         473  +
    attempt_time.advance(Duration::from_secs(10));
         474  +
    response_gate.release();
         475  +
    let (status, body) = send.await.expect("request task");
         476  +
    assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         477  +
         478  +
    let telemetry = capture.get();
         479  +
    assert_eq!(
         480  +
        telemetry.acquisition().expect("acquisition").duration(),
         481  +
        Some(Duration::ZERO)
         482  +
    );
         483  +
    assert_eq!(
         484  +
        telemetry.connector_call_duration(),
         485  +
        Some(Duration::from_secs(10))
         486  +
    );
         487  +
         488  +
    drop(connector);
         489  +
    drop(client);
         490  +
    drop(pool);
         491  +
    harness.shutdown().await.expect("clean harness shutdown");
         492  +
}
         493  +
  322    494   
#[tokio::test]
  323    495   
async fn bounded_waiter_proceeds_after_the_active_h1_returns() {
  324    496   
    let body_gate = ManualGate::new();
  325    497   
    let harness = ConnectionTestHarness::builder()
  326    498   
        .endpoint(
  327    499   
            IP1,
  328    500   
            Http1Script::responses([
  329    501   
                Http1Response::ok().body_plan(BodyPlan::split_at_gate(
  330    502   
                    "first-",
  331    503   
                    body_gate.waiter(),
  332    504   
                    "body",
  333    505   
                )),
  334    506   
                Http1Response::ok().body("second"),
  335    507   
            ]),
  336    508   
        )
  337    509   
        .build()
  338    510   
        .await
  339    511   
        .expect("harness should start");
         512  +
    let attempt_time = ManualTimeSource::new(UNIX_EPOCH);
  340    513   
    let pool = ConnectionPool::builder()
  341    514   
        .max_connections_per_host(1)
  342    515   
        .build_http()
  343    516   
        .expect("valid pool");
  344    517   
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
  345         -
    let connector = connector(&client);
         518  +
    let connector = test_client::connector_with_time_source(&client, attempt_time.clone());
  346    519   
  347    520   
    let first = test_client::send_request(
  348    521   
        &connector,
  349    522   
        HttpRequest::get(harness.endpoint_url()).expect("valid request"),
  350    523   
    )
  351    524   
    .await
  352    525   
    .expect("first request should reach response headers");
  353    526   
    body_gate
  354    527   
        .wait_until_reached(test_client::WAIT)
  355    528   
        .await
  356    529   
        .expect("first response should reach its body gate");
  357    530   
         531  +
    let second_capture = CaptureHttpAttemptTelemetry::new();
         532  +
    let mut second_request = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         533  +
    second_request.add_extension(second_capture.clone());
  358    534   
    let second_connector = connector.clone();
  359         -
    let second_url = harness.endpoint_url();
  360         -
    let mut second =
  361         -
        tokio::spawn(
  362         -
            async move { test_client::get_and_collect(&second_connector, &second_url).await },
  363         -
        );
         535  +
    let mut second = tokio::spawn(async move {
         536  +
        test_client::send_and_collect(&second_connector, second_request).await
         537  +
    });
  364    538   
    assert!(
  365    539   
        tokio::time::timeout(Duration::from_millis(100), &mut second)
  366    540   
            .await
  367    541   
            .is_err(),
  368    542   
        "the second request must wait while the only admitted H1 is active"
  369    543   
    );
  370    544   
    assert_eq!(1, harness.tcp_accepted_count());
  371    545   
         546  +
    attempt_time.advance(Duration::from_secs(5));
  372    547   
    body_gate.release();
  373    548   
    let (status, body) = test_client::collect_response(first).await;
  374    549   
    assert_eq!((status, body.as_slice()), (200, b"first-body".as_slice()));
  375    550   
    let (status, body) = second.await.expect("second request task should not panic");
  376    551   
    assert_eq!((status, body.as_slice()), (200, b"second".as_slice()));
  377    552   
    assert_eq!(1, harness.tcp_accepted_count());
         553  +
    let telemetry = second_capture.get();
         554  +
    let acquisition = telemetry.acquisition().expect("second acquisition");
         555  +
    assert_eq!(acquisition.duration(), Some(Duration::from_secs(5)));
         556  +
    assert_eq!(acquisition.usage(), ConnectionUsage::Reused);
         557  +
    assert_eq!(
         558  +
        telemetry.connector_call_duration(),
         559  +
        Some(Duration::from_secs(5))
         560  +
    );
  378    561   
  379    562   
    drop(connector);
  380    563   
    drop(client);
  381    564   
    drop(pool);
  382    565   
    harness.shutdown().await.expect("clean harness shutdown");
  383    566   
}
  384    567   
  385    568   
#[tokio::test]
  386    569   
async fn eligible_partition_borrows_the_peer_h1() {
  387    570   
    let harness = ConnectionTestHarness::builder()
@@ -410,593 +469,834 @@
  430    613   
    assert_eq!(1, harness.tcp_accepted_count());
  431    614   
  432    615   
    drop(first_connector);
  433    616   
    drop(second_connector);
  434    617   
    drop(first_client);
  435    618   
    drop(second_client);
  436    619   
    drop(pool);
  437    620   
    harness.shutdown().await.expect("clean harness shutdown");
  438    621   
}
  439    622   
         623  +
/// Request body that remains open until the test releases it.
         624  +
struct HeldUpload {
         625  +
    finish: oneshot::Receiver<()>,
         626  +
    complete: bool,
         627  +
}
         628  +
         629  +
impl Body for HeldUpload {
         630  +
    type Data = Bytes;
         631  +
    type Error = Infallible;
         632  +
         633  +
    fn poll_frame(
         634  +
        mut self: Pin<&mut Self>,
         635  +
        cx: &mut Context<'_>,
         636  +
    ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
         637  +
        if self.complete {
         638  +
            return Poll::Ready(None);
         639  +
        }
         640  +
        match Pin::new(&mut self.finish).poll(cx) {
         641  +
            Poll::Ready(_) => {
         642  +
                self.complete = true;
         643  +
                Poll::Ready(None)
         644  +
            }
         645  +
            Poll::Pending => Poll::Pending,
         646  +
        }
         647  +
    }
         648  +
         649  +
    fn is_end_stream(&self) -> bool {
         650  +
        self.complete
         651  +
    }
         652  +
         653  +
    fn size_hint(&self) -> SizeHint {
         654  +
        SizeHint::default()
         655  +
    }
         656  +
}
         657  +
         658  +
fn held_upload() -> (oneshot::Sender<()>, SdkBody) {
         659  +
    let (finish, finished) = oneshot::channel();
         660  +
    (
         661  +
        finish,
         662  +
        SdkBody::from_body_1_x(HeldUpload {
         663  +
            finish: finished,
         664  +
            complete: false,
         665  +
        }),
         666  +
    )
         667  +
}
         668  +
         669  +
#[tokio::test]
         670  +
async fn peer_h1_reuse_survives_borrower_runtime_shutdown() {
         671  +
    use runtime::DrivenRuntime;
         672  +
         673  +
    let peer_response_gate = ManualGate::new();
         674  +
    let harness = ConnectionTestHarness::builder()
         675  +
        .endpoint(
         676  +
            IP1,
         677  +
            SocketScript::new()
         678  +
                .read_http1_request()
         679  +
                .write_all(
         680  +
                    b"HTTP/1.1 200 OK\r\n\
         681  +
                      Content-Length: 5\r\n\
         682  +
                      Connection: keep-alive\r\n\
         683  +
                      \r\n\
         684  +
                      owner",
         685  +
                )
         686  +
                .read_until(b"\r\n\r\n", 16 * 1024)
         687  +
                .wait(peer_response_gate.waiter())
         688  +
                .write_all(
         689  +
                    b"HTTP/1.1 200 OK\r\n\
         690  +
                      Content-Length: 4\r\n\
         691  +
                      Connection: keep-alive\r\n\
         692  +
                      \r\n\
         693  +
                      peer",
         694  +
                )
         695  +
                .read_until(b"0\r\n\r\n", 1024)
         696  +
                .read_http1_request()
         697  +
                .write_all(
         698  +
                    b"HTTP/1.1 200 OK\r\n\
         699  +
                      Content-Length: 6\r\n\
         700  +
                      Connection: keep-alive\r\n\
         701  +
                      \r\n\
         702  +
                      reused",
         703  +
                )
         704  +
                .await_client_close(),
         705  +
        )
         706  +
        .build()
         707  +
        .await
         708  +
        .expect("harness should start");
         709  +
    let first_id = PartitionId::from_index(1);
         710  +
    let second_id = PartitionId::from_index(2);
         711  +
    let first_runtime = DrivenRuntime::start("h1-owner-one");
         712  +
    let second_runtime = DrivenRuntime::start("h1-owner-two");
         713  +
    assert_ne!(first_runtime.id(), second_runtime.id());
         714  +
    let pool = ConnectionPool::builder()
         715  +
        .partitions([
         716  +
            Partition::new(first_id, first_runtime.driver_spawner()),
         717  +
            Partition::new(second_id, second_runtime.driver_spawner()),
         718  +
        ])
         719  +
        .connection_reuse_scope(ConnectionReuseScope::Pool)
         720  +
        .max_connections_per_host(1)
         721  +
        .build_http()
         722  +
        .expect("valid pool");
         723  +
    let first_client = shared_client(&pool, first_id);
         724  +
    let second_client = shared_client(&pool, second_id);
         725  +
    let first_connector = connector(&first_client);
         726  +
    let second_connector = connector(&second_client);
         727  +
    let url = harness.endpoint_url();
         728  +
         729  +
    let first_request = first_connector.clone();
         730  +
    let first_url = url.clone();
         731  +
    let (status, body) = first_runtime
         732  +
        .spawn(async move { test_client::get_and_collect(&first_request, &first_url).await })
         733  +
        .await
         734  +
        .expect("owner request task should not panic");
         735  +
    assert_eq!((status, body.as_slice()), (200, b"owner".as_slice()));
         736  +
         737  +
    let (finish_upload, upload) = held_upload();
         738  +
    let mut peer_request = HttpRequest::new(upload);
         739  +
    peer_request
         740  +
        .set_uri(url.clone())
         741  +
        .expect("valid HTTP request URI");
         742  +
    peer_request.set_method("POST").expect("valid HTTP method");
         743  +
    let peer_connector = second_connector.clone();
         744  +
    let peer = second_runtime
         745  +
        .spawn(async move { test_client::send_and_collect(&peer_connector, peer_request).await });
         746  +
    peer_response_gate
         747  +
        .wait_until_reached(test_client::WAIT)
         748  +
        .await
         749  +
        .expect("peer request should reach the response gate");
         750  +
    let owner_tasks_before_peer_completion = first_runtime.submitted_tasks();
         751  +
    let borrower_tasks_before_peer_completion = second_runtime.submitted_tasks();
         752  +
    peer_response_gate.release();
         753  +
    let (status, body) = peer.await.expect("peer request task should not panic");
         754  +
    assert_eq!((status, body.as_slice()), (200, b"peer".as_slice()));
         755  +
    assert_eq!(
         756  +
        owner_tasks_before_peer_completion + 1,
         757  +
        first_runtime.submitted_tasks(),
         758  +
        "deferred H1 readiness must use the connection owner's spawner"
         759  +
    );
         760  +
    assert_eq!(
         761  +
        borrower_tasks_before_peer_completion,
         762  +
        second_runtime.submitted_tasks(),
         763  +
        "the borrowing partition must not own readiness for a peer connection"
         764  +
    );
         765  +
    finish_upload
         766  +
        .send(())
         767  +
        .expect("peer upload should remain open until readiness placement is observed");
         768  +
         769  +
    second_runtime.shutdown();
         770  +
         771  +
    let reused_request = first_connector.clone();
         772  +
    let reused_url = url.clone();
         773  +
    let (status, body) = first_runtime
         774  +
        .spawn(async move { test_client::get_and_collect(&reused_request, &reused_url).await })
         775  +
        .await
         776  +
        .expect("reused request task should not panic");
         777  +
    assert_eq!((status, body.as_slice()), (200, b"reused".as_slice()));
         778  +
         779  +
    let request_connections = harness
         780  +
        .events()
         781  +
        .into_iter()
         782  +
        .filter_map(|event| match event {
         783  +
            WireConnectionEvent::Http1Request { connection_id, .. } => Some(connection_id),
         784  +
            _ => None,
         785  +
        })
         786  +
        .collect::<Vec<_>>();
         787  +
    assert_eq!(2, request_connections.len());
         788  +
    assert!(
         789  +
        request_connections
         790  +
            .iter()
         791  +
            .all(|connection| *connection == request_connections[0]),
         792  +
        "owner and peer requests should use one HTTP/1 connection"
         793  +
    );
         794  +
    assert_eq!(1, harness.tcp_accepted_count());
         795  +
         796  +
    drop(first_connector);
         797  +
    drop(second_connector);
         798  +
    drop(first_client);
         799  +
    drop(second_client);
         800  +
    drop(pool);
         801  +
    first_runtime.shutdown();
         802  +
    harness.shutdown().await.expect("clean harness shutdown");
         803  +
}
         804  +
  440    805   
#[tokio::test]
  441    806   
async fn ineligible_partition_reclaims_peer_capacity() {
  442    807   
    let harness = ConnectionTestHarness::builder()
  443    808   
        .endpoint(
  444    809   
            IP1,
  445    810   
            EndpointPlan::queue([
  446    811   
                Http1Script::responses([Http1Response::ok().body("first")]),
  447    812   
                Http1Script::responses([Http1Response::ok().body("second")]),
  448    813   
            ]),
  449    814   
        )

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/h2_connection_behavior_test.rs

@@ -1,1 +53,58 @@
    4      4   
 */
    5      5   
    6      6   
//! HTTP/2 connection behavior contracts.
    7      7   
//!
    8      8   
//! Each contract is implementation-neutral and has an explicit runner for every
    9      9   
//! HTTP client backend expected to preserve that behavior.
   10     10   
   11     11   
#![cfg(all(feature = "wire-mock", feature = "rustls-aws-lc"))]
   12     12   
   13     13   
mod common;
          14  +
#[path = "common/runtime.rs"]
          15  +
mod runtime;
   14     16   
   15     17   
use aws_smithy_http_client::pool::{
   16     18   
    Client as PoolClient, ConnectionPool, ConnectionReuseScope, Partition, PartitionId,
   17     19   
    TokioDriverSpawner,
   18     20   
};
   19     21   
use aws_smithy_http_client::test_util::wire::connection::{ConnectionCloseReason, ManualGate};
   20     22   
use aws_smithy_http_client::tls;
   21     23   
use aws_smithy_runtime_api::client::connection::{
   22     24   
    CaptureSmithyConnection, ConnectionMetadata as SmithyConnectionMetadata,
   23     25   
};
          26  +
use aws_smithy_runtime_api::client::http::telemetry::{
          27  +
    CaptureHttpAttemptTelemetry, ConnectionUsage,
          28  +
};
   24     29   
use aws_smithy_runtime_api::client::http::{SharedHttpClient, SharedHttpConnector};
   25     30   
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
   26     31   
use bytes::Bytes;
   27     32   
use common::client as test_client;
   28     33   
use common::client::{
   29     34   
    BackendConfig, HttpsClientBackend, HyperUtilLegacyPool, PartitionedConnectionPool,
   30     35   
};
   31     36   
use common::h2::{
   32     37   
    H2BodyPlan, H2ConnectionId, H2ConnectionPlan, H2ConnectionScript, H2Event, H2Response,
   33     38   
    H2StreamScript, H2TestServer,
@@ -248,253 +307,443 @@
  268    273   
    }
  269    274   
  270    275   
    #[tokio::test]
  271    276   
    async fn test_concurrent_cold_start_converges_on_one_h2_connection_with_partitioned_pool() {
  272    277   
        concurrent_cold_start_converges_on_one_h2_connection(&PartitionedConnectionPool).await;
  273    278   
    }
  274    279   
}
  275    280   
  276    281   
mod connection_metadata {
  277    282   
    use super::*;
         283  +
    use aws_smithy_async::test_util::ManualTimeSource;
         284  +
    use std::time::{Duration, UNIX_EPOCH};
         285  +
         286  +
    async fn capture_attempt(
         287  +
        connector: &SharedHttpConnector,
         288  +
        url: &str,
         289  +
    ) -> aws_smithy_runtime_api::client::http::telemetry::HttpAttemptTelemetry {
         290  +
        let capture = CaptureHttpAttemptTelemetry::new();
         291  +
        let mut request = HttpRequest::get(url).expect("valid HTTP request");
         292  +
        request.add_extension(capture.clone());
         293  +
        let (status, body) = test_client::send_and_collect(connector, request).await;
         294  +
        assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         295  +
        capture.get()
         296  +
    }
         297  +
         298  +
    async fn attempt_telemetry_matches_backend_capability(
         299  +
        backend: &dyn HttpsClientBackend,
         300  +
        captures_selection: bool,
         301  +
    ) {
         302  +
        let server = H2TestServer::builder()
         303  +
            .connections(H2ConnectionPlan::queue([
         304  +
                H2ConnectionScript::new().fallback(H2StreamScript::respond(H2Response::ok("ok")))
         305  +
            ]))
         306  +
            .start()
         307  +
            .await
         308  +
            .expect("H2 server should start");
         309  +
        let client = h2_client(backend);
         310  +
        let connector = test_client::connector(&client);
         311  +
         312  +
        let first = capture_attempt(&connector, &server.url("/first")).await;
         313  +
        let second = capture_attempt(&connector, &server.url("/second")).await;
         314  +
        assert!(first.connector_call_duration().is_some());
         315  +
        assert!(second.connector_call_duration().is_some());
         316  +
         317  +
        if captures_selection {
         318  +
            assert_eq!(
         319  +
                first.acquisition().expect("first acquisition").usage(),
         320  +
                ConnectionUsage::Fresh
         321  +
            );
         322  +
            assert_eq!(
         323  +
                second.acquisition().expect("second acquisition").usage(),
         324  +
                ConnectionUsage::Reused
         325  +
            );
         326  +
            let first_connection = first.connection().expect("first connection metadata");
         327  +
            let second_connection = second.connection().expect("second connection metadata");
         328  +
            assert_eq!(
         329  +
                first_connection.connection_id(),
         330  +
                second_connection.connection_id()
         331  +
            );
         332  +
            assert_eq!(
         333  +
                first_connection.establishment(),
         334  +
                second_connection.establishment(),
         335  +
                "reuse must retain the establishment that created the connection"
         336  +
            );
         337  +
            assert!(first_connection.establishment().is_some());
         338  +
        } else {
         339  +
            assert!(first.acquisition().is_none());
         340  +
            assert!(first.connection().is_none());
         341  +
            assert!(second.acquisition().is_none());
         342  +
            assert!(second.connection().is_none());
         343  +
        }
         344  +
         345  +
        drop(connector);
         346  +
        drop(client);
         347  +
        server.shutdown().await.expect("clean H2 server shutdown");
         348  +
    }
         349  +
         350  +
    #[tokio::test]
         351  +
    async fn test_attempt_telemetry_with_hyper_util_legacy_pool() {
         352  +
        attempt_telemetry_matches_backend_capability(&HyperUtilLegacyPool, false).await;
         353  +
    }
         354  +
         355  +
    #[tokio::test]
         356  +
    async fn test_attempt_telemetry_with_partitioned_pool() {
         357  +
        attempt_telemetry_matches_backend_capability(&PartitionedConnectionPool, true).await;
         358  +
    }
         359  +
         360  +
    #[tokio::test]
         361  +
    async fn h2_acquisition_ends_before_response_headers() {
         362  +
        let response_gate = ManualGate::new();
         363  +
        let server = H2TestServer::builder()
         364  +
            .connections(H2ConnectionPlan::queue([H2ConnectionScript::new()
         365  +
                .fallback(H2StreamScript::respond_after(
         366  +
                    H2Response::ok("ok"),
         367  +
                    response_gate.waiter(),
         368  +
                ))]))
         369  +
            .start()
         370  +
            .await
         371  +
            .expect("H2 server should start");
         372  +
        let time = ManualTimeSource::new(UNIX_EPOCH);
         373  +
        let pool = ConnectionPool::builder()
         374  +
            .tls_provider(rustls_aws_lc())
         375  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
         376  +
            .time_source(time.clone())
         377  +
            .build_https()
         378  +
            .expect("valid partitioned HTTPS pool");
         379  +
        let client =
         380  +
            SharedHttpClient::new(PoolClient::new(&pool).expect("anonymous partition exists"));
         381  +
        let connector = test_client::connector_with_time_source(&client, time.clone());
         382  +
        let capture = CaptureHttpAttemptTelemetry::new();
         383  +
        let mut request = HttpRequest::get(server.url("/gated")).expect("valid HTTP request");
         384  +
        request.add_extension(capture.clone());
         385  +
        let send = tokio::spawn({
         386  +
            let connector = connector.clone();
         387  +
            async move { test_client::send_and_collect(&connector, request).await }
         388  +
        });
         389  +
         390  +
        response_gate
         391  +
            .wait_until_reached(test_client::WAIT)
         392  +
            .await
         393  +
            .expect("server should accept the H2 stream");
         394  +
        time.advance(Duration::from_secs(10));
         395  +
        response_gate.release();
         396  +
        let (status, body) = send.await.expect("request task");
         397  +
        assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         398  +
         399  +
        let telemetry = capture.get();
         400  +
        assert_eq!(
         401  +
            telemetry.acquisition().expect("acquisition").duration(),
         402  +
            Some(Duration::ZERO)
         403  +
        );
         404  +
        assert_eq!(
         405  +
            telemetry.connector_call_duration(),
         406  +
            Some(Duration::from_secs(10))
         407  +
        );
         408  +
         409  +
        drop(connector);
         410  +
        drop(client);
         411  +
        drop(pool);
         412  +
        server.shutdown().await.expect("clean H2 server shutdown");
         413  +
    }
  278    414   
  279    415   
    /// Poisoning captured H2 connection metadata moves later streams to a new connection.
  280    416   
    async fn poisoned_connection_is_not_reused(backend: &dyn HttpsClientBackend) {
  281    417   
        let script =
  282    418   
            H2ConnectionScript::new().fallback(H2StreamScript::respond(H2Response::ok("ok")));
  283    419   
        let server = H2TestServer::builder()
  284    420   
            .connections(H2ConnectionPlan::queue([script.clone(), script]))
  285    421   
            .start()
  286    422   
            .await
  287    423   
            .expect("H2 server should start");
@@ -612,748 +671,1050 @@
  632    768   
            "GOAWAY before dispatch created {} connections",
  633    769   
            server.connection_count()
  634    770   
        );
  635    771   
  636    772   
        drop(connector);
  637    773   
        drop(client);
  638    774   
        server.shutdown().await.expect("clean H2 server shutdown");
  639    775   
    }
  640    776   
}
  641    777   
         778  +
mod abrupt_transport_failure {
         779  +
    use super::*;
         780  +
    use aws_smithy_http_client::pool::ConnectionEvent;
         781  +
    use aws_smithy_runtime_api::client::connection::ConnectionId;
         782  +
    use std::sync::{Arc, Mutex};
         783  +
    use std::time::Duration;
         784  +
         785  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
         786  +
    enum PoolEvent {
         787  +
        Opened(ConnectionId),
         788  +
        LogicalClose(ConnectionId),
         789  +
        PhysicalClose(ConnectionId),
         790  +
    }
         791  +
         792  +
    /// Raw transport loss terminates every accepted stream and later demand uses
         793  +
    /// a replacement connection.
         794  +
    async fn abrupt_transport_failure_terminates_streams_and_replaces_connection(
         795  +
        client: SharedHttpClient,
         796  +
    ) -> aws_smithy_http_client::pool::OriginKey {
         797  +
        let body_gate = ManualGate::new();
         798  +
        let held = |body| {
         799  +
            H2StreamScript::respond(
         800  +
                H2Response::new(http_1x::StatusCode::OK).body(H2BodyPlan::gated(
         801  +
                    body,
         802  +
                    body_gate.waiter(),
         803  +
                    "-never",
         804  +
                )),
         805  +
            )
         806  +
        };
         807  +
        let first = H2ConnectionScript::new()
         808  +
            .route("/warm", H2StreamScript::respond(H2Response::ok("warm")))
         809  +
            .route("/one", held("one"))
         810  +
            .route("/two", held("two"))
         811  +
            .route("/three", held("three"));
         812  +
        let second = H2ConnectionScript::new().route(
         813  +
            "/replacement",
         814  +
            H2StreamScript::respond(H2Response::ok("replacement")),
         815  +
        );
         816  +
        let server = H2TestServer::builder()
         817  +
            .connections(H2ConnectionPlan::queue([first, second]))
         818  +
            .start()
         819  +
            .await
         820  +
            .expect("H2 server should start");
         821  +
        let connector = test_client::connector(&client);
         822  +
         823  +
        let (status, body) = test_client::get_and_collect(&connector, &server.url("/warm")).await;
         824  +
        assert_eq!((status, body.as_slice()), (200, b"warm".as_slice()));
         825  +
        let (one, two, three) = tokio::join!(
         826  +
            test_client::send_request(
         827  +
                &connector,
         828  +
                HttpRequest::get(server.url("/one")).expect("valid HTTP request"),
         829  +
            ),
         830  +
            test_client::send_request(
         831  +
                &connector,
         832  +
                HttpRequest::get(server.url("/two")).expect("valid HTTP request"),
         833  +
            ),
         834  +
            test_client::send_request(
         835  +
                &connector,
         836  +
                HttpRequest::get(server.url("/three")).expect("valid HTTP request"),
         837  +
            ),
         838  +
        );
         839  +
        let responses = [
         840  +
            one.expect("first response headers should arrive"),
         841  +
            two.expect("second response headers should arrive"),
         842  +
            three.expect("third response headers should arrive"),
         843  +
        ];
         844  +
        body_gate
         845  +
            .wait_for_arrivals(3, test_client::WAIT)
         846  +
            .await
         847  +
            .expect("all response streams should reach the body gate");
         848  +
         849  +
        let original = single_stream_connection(&server, "/warm");
         850  +
        assert_eq!(original, single_stream_connection(&server, "/one"));
         851  +
        assert_eq!(original, single_stream_connection(&server, "/two"));
         852  +
        assert_eq!(original, single_stream_connection(&server, "/three"));
         853  +
        server
         854  +
            .abort_transport(original)
         855  +
            .await
         856  +
            .expect("the original transport should abort");
         857  +
         858  +
        for response in responses {
         859  +
            let body_result =
         860  +
                tokio::time::timeout(test_client::WAIT, response.into_body().collect())
         861  +
                    .await
         862  +
                    .expect("aborted response body should terminate");
         863  +
            assert!(
         864  +
                body_result.is_err(),
         865  +
                "abrupt transport loss completed an accepted response body"
         866  +
            );
         867  +
        }
         868  +
        server
         869  +
            .wait_for_event(test_client::WAIT, |event| {
         870  +
                matches!(
         871  +
                    event,
         872  +
                    H2Event::ConnectionClosed {
         873  +
                        connection_id,
         874  +
                        reason: ConnectionCloseReason::ScriptedTransportAbort,
         875  +
                    } if *connection_id == original
         876  +
                )
         877  +
            })
         878  +
            .await
         879  +
            .expect("the harness should record raw transport loss");
         880  +
         881  +
        let replacement_url = server.url("/replacement");
         882  +
        let origin = aws_smithy_http_client::pool::OriginKey::from_uri(
         883  +
            &replacement_url.parse().expect("valid replacement URI"),
         884  +
        )
         885  +
        .expect("replacement URI should name an origin");
         886  +
        let (status, body) = test_client::get_and_collect(&connector, &replacement_url).await;
         887  +
        assert_eq!((status, body.as_slice()), (200, b"replacement".as_slice()));
         888  +
        assert_ne!(
         889  +
            original,
         890  +
            single_stream_connection(&server, "/replacement"),
         891  +
            "later demand reused the aborted connection"
         892  +
        );
         893  +
        assert_eq!(2, server.connection_count());
         894  +
         895  +
        drop(connector);
         896  +
        drop(client);
         897  +
        server.shutdown().await.expect("clean H2 server shutdown");
         898  +
        origin
         899  +
    }
         900  +
         901  +
    async fn wait_for_pool_events(
         902  +
        events: &Arc<Mutex<Vec<PoolEvent>>>,
         903  +
        predicate: impl Fn(&[PoolEvent]) -> bool,
         904  +
    ) {
         905  +
        tokio::time::timeout(test_client::WAIT, async {
         906  +
            loop {
         907  +
                if predicate(&events.lock().unwrap()) {
         908  +
                    return;
         909  +
                }
         910  +
                tokio::time::sleep(Duration::from_millis(10)).await;
         911  +
            }
         912  +
        })
         913  +
        .await
         914  +
        .expect("pool lifecycle events should converge");
         915  +
    }
         916  +
         917  +
    #[tokio::test]
         918  +
    async fn test_abrupt_transport_failure_terminates_streams_and_replaces_connection_with_hyper_util_legacy_pool(
         919  +
    ) {
         920  +
        abrupt_transport_failure_terminates_streams_and_replaces_connection(h2_client(
         921  +
            &HyperUtilLegacyPool,
         922  +
        ))
         923  +
        .await;
         924  +
    }
         925  +
         926  +
    #[tokio::test]
         927  +
    async fn test_abrupt_transport_failure_terminates_streams_and_replaces_connection_with_partitioned_pool(
         928  +
    ) {
         929  +
        let events = Arc::new(Mutex::new(Vec::new()));
         930  +
        let observed = events.clone();
         931  +
        let pool = ConnectionPool::builder()
         932  +
            .tls_provider(rustls_aws_lc())
         933  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
         934  +
            .max_connections_per_host(1)
         935  +
            .event_listener(move |event: &ConnectionEvent<'_>| {
         936  +
                let event = match event {
         937  +
                    ConnectionEvent::Opened(opened) => {
         938  +
                        Some(PoolEvent::Opened(opened.connection().id()))
         939  +
                    }
         940  +
                    ConnectionEvent::LogicalClose(closed) => {
         941  +
                        Some(PoolEvent::LogicalClose(closed.connection().id()))
         942  +
                    }
         943  +
                    ConnectionEvent::PhysicalClose(closed) => {
         944  +
                        Some(PoolEvent::PhysicalClose(closed.connection().id()))
         945  +
                    }
         946  +
                    ConnectionEvent::EstablishmentFailed(_) => None,
         947  +
                    _ => None,
         948  +
                };
         949  +
                if let Some(event) = event {
         950  +
                    observed.lock().unwrap().push(event);
         951  +
                }
         952  +
            })
         953  +
            .build_https()
         954  +
            .expect("valid partitioned HTTPS pool");
         955  +
        let client = SharedHttpClient::new(
         956  +
            PoolClient::new(&pool).expect("anonymous partition should resolve"),
         957  +
        );
         958  +
         959  +
        let origin =
         960  +
            abrupt_transport_failure_terminates_streams_and_replaces_connection(client).await;
         961  +
        wait_for_pool_events(&events, |events| {
         962  +
            events
         963  +
                .iter()
         964  +
                .filter(|event| matches!(event, PoolEvent::PhysicalClose(_)))
         965  +
                .count()
         966  +
                >= 1
         967  +
        })
         968  +
        .await;
         969  +
         970  +
        let events = events.lock().unwrap();
         971  +
        let opened: Vec<_> = events
         972  +
            .iter()
         973  +
            .filter_map(|event| match event {
         974  +
                PoolEvent::Opened(connection) => Some(*connection),
         975  +
                _ => None,
         976  +
            })
         977  +
            .collect();
         978  +
        assert_eq!(2, opened.len(), "expected original and replacement opens");
         979  +
        assert_ne!(opened[0], opened[1]);
         980  +
        assert_eq!(
         981  +
            1,
         982  +
            events
         983  +
                .iter()
         984  +
                .filter(|event| matches!(event, PoolEvent::LogicalClose(id) if *id == opened[0]))
         985  +
                .count(),
         986  +
            "the aborted generation should close logically once"
         987  +
        );
         988  +
        assert_eq!(
         989  +
            1,
         990  +
            events
         991  +
                .iter()
         992  +
                .filter(|event| matches!(event, PoolEvent::PhysicalClose(id) if *id == opened[0]))
         993  +
                .count(),
         994  +
            "the aborted transport should close physically once"
         995  +
        );
         996  +
        drop(events);
         997  +
         998  +
        let stats = pool
         999  +
            .partition_stats(PartitionId::ANONYMOUS, &origin)
        1000  +
            .expect("anonymous partition should retain origin statistics");
        1001  +
        assert_eq!(0, stats.pending_acquisitions());
        1002  +
        assert_eq!(0, stats.establishing_connections());
        1003  +
        assert_eq!(0, stats.h2().active_requests());
        1004  +
        assert_eq!(1, stats.h2().accepting());
        1005  +
        assert_eq!(0, stats.h2().draining());
        1006  +
        assert_eq!(1, stats.physically_live_connections());
        1007  +
        let capacity = pool
        1008  +
            .origin_stats(&origin)
        1009  +
            .capacity()
        1010  +
            .copied()
        1011  +
            .expect("origin should have bounded capacity");
        1012  +
        assert_eq!(1, capacity.limit());
        1013  +
        assert_eq!(
        1014  +
            1,
        1015  +
            capacity.in_use(),
        1016  +
            "replacement connection should own the returned slot"
        1017  +
        );
        1018  +
    }
        1019  +
}
        1020  +
  642   1021   
#[cfg(feature = "s2n-tls")]
  643   1022   
mod protocol_negotiation {
  644   1023   
    use super::*;
  645   1024   
  646   1025   
    /// The s2n-tls provider negotiates `h2` with ALPN and reuses that connection.
  647   1026   
    async fn s2n_negotiates_h2_and_reuses_connection(backend: &dyn HttpsClientBackend) {
  648   1027   
        let server = H2TestServer::builder()
  649   1028   
            .connections(H2ConnectionPlan::queue([H2ConnectionScript::new()
  650   1029   
                .fallback(H2StreamScript::respond(H2Response::ok("s2n-h2")))]))
  651   1030   
            .start()
@@ -1052,1431 +0,1697 @@
 1072   1451   
        let second_connection = single_stream_connection(&server, "/second");
 1073   1452   
        assert_ne!(first_connection, second_connection);
 1074   1453   
        assert_eq!(server.connection_count(), 2);
 1075   1454   
 1076   1455   
        drop(metadata);
 1077   1456   
        drop(first);
 1078   1457   
        drop(pool);
 1079   1458   
        server.shutdown().await.expect("clean H2 server shutdown");
 1080   1459   
    }
 1081   1460   
}
        1461  +
        1462  +
mod runtime_placement {
        1463  +
    use super::*;
        1464  +
    use aws_smithy_http_client::pool::ConnectionEvent;
        1465  +
    use aws_smithy_runtime_api::client::connection::ConnectionId;
        1466  +
    use runtime::DrivenRuntime;
        1467  +
    use std::sync::{Arc, Mutex};
        1468  +
    use std::time::Duration;
        1469  +
    use tokio::runtime::{Handle, Id};
        1470  +
        1471  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
        1472  +
    enum EventKind {
        1473  +
        Opened,
        1474  +
        LogicalClose,
        1475  +
        PhysicalClose,
        1476  +
    }
        1477  +
        1478  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
        1479  +
    struct ObservedEvent {
        1480  +
        kind: EventKind,
        1481  +
        connection: ConnectionId,
        1482  +
        partition: PartitionId,
        1483  +
        runtime: Option<Id>,
        1484  +
    }
        1485  +
        1486  +
    async fn wait_for_event(
        1487  +
        events: &Arc<Mutex<Vec<ObservedEvent>>>,
        1488  +
        predicate: impl Fn(&ObservedEvent) -> bool,
        1489  +
    ) -> ObservedEvent {
        1490  +
        tokio::time::timeout(test_client::WAIT, async {
        1491  +
            loop {
        1492  +
                if let Some(event) = events.lock().unwrap().iter().copied().find(&predicate) {
        1493  +
                    return event;
        1494  +
                }
        1495  +
                tokio::time::sleep(Duration::from_millis(10)).await;
        1496  +
            }
        1497  +
        })
        1498  +
        .await
        1499  +
        .expect("connection event should arrive")
        1500  +
    }
        1501  +
        1502  +
    #[tokio::test]
        1503  +
    async fn peer_h2_reuse_keeps_connection_work_on_the_owner_runtime() {
        1504  +
        let first_id = PartitionId::from_index(1);
        1505  +
        let second_id = PartitionId::from_index(2);
        1506  +
        let first_runtime = DrivenRuntime::start("h2-owner-one");
        1507  +
        let first_runtime_id = first_runtime.id();
        1508  +
        let second_runtime = DrivenRuntime::start("h2-owner-two");
        1509  +
        let second_runtime_id = second_runtime.id();
        1510  +
        let peer_body_gate = ManualGate::new();
        1511  +
        let first_script = H2ConnectionScript::new()
        1512  +
            .abort_streams_on_client_close()
        1513  +
            .route("/first", H2StreamScript::respond(H2Response::ok("first")))
        1514  +
            .route(
        1515  +
                "/peer",
        1516  +
                H2StreamScript::respond(
        1517  +
                    H2Response::new(http_1x::StatusCode::OK).body(H2BodyPlan::gated(
        1518  +
                        "peer-",
        1519  +
                        peer_body_gate.waiter(),
        1520  +
                        "never",
        1521  +
                    )),
        1522  +
                ),
        1523  +
            );
        1524  +
        let second_script = H2ConnectionScript::new().route(
        1525  +
            "/replacement",
        1526  +
            H2StreamScript::respond(H2Response::ok("replacement")),
        1527  +
        );
        1528  +
        let server = H2TestServer::builder()
        1529  +
            .connections(H2ConnectionPlan::queue([first_script, second_script]))
        1530  +
            .start()
        1531  +
            .await
        1532  +
            .expect("H2 server should start");
        1533  +
        1534  +
        let events = Arc::new(Mutex::new(Vec::new()));
        1535  +
        let observed = events.clone();
        1536  +
        let pool = ConnectionPool::builder()
        1537  +
            .tls_provider(rustls_aws_lc())
        1538  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
        1539  +
            .partitions([
        1540  +
                Partition::new(first_id, first_runtime.driver_spawner()),
        1541  +
                Partition::new(second_id, second_runtime.driver_spawner()),
        1542  +
            ])
        1543  +
            .connection_reuse_scope(ConnectionReuseScope::Pool)
        1544  +
            .max_connections_per_host(1)
        1545  +
            .event_listener(move |event: &ConnectionEvent<'_>| {
        1546  +
                let observed_event = match event {
        1547  +
                    ConnectionEvent::Opened(opened) => {
        1548  +
                        Some((EventKind::Opened, opened.connection()))
        1549  +
                    }
        1550  +
                    ConnectionEvent::LogicalClose(closed) => {
        1551  +
                        Some((EventKind::LogicalClose, closed.connection()))
        1552  +
                    }
        1553  +
                    ConnectionEvent::PhysicalClose(closed) => {
        1554  +
                        Some((EventKind::PhysicalClose, closed.connection()))
        1555  +
                    }
        1556  +
                    ConnectionEvent::EstablishmentFailed(_) => None,
        1557  +
                    _ => None,
        1558  +
                };
        1559  +
                if let Some((kind, connection)) = observed_event {
        1560  +
                    observed.lock().unwrap().push(ObservedEvent {
        1561  +
                        kind,
        1562  +
                        connection: connection.id(),
        1563  +
                        partition: connection.owner_partition(),
        1564  +
                        runtime: Handle::try_current().ok().map(|handle| handle.id()),
        1565  +
                    });
        1566  +
                }
        1567  +
            })
        1568  +
            .build_https()
        1569  +
            .expect("valid partitioned HTTPS pool");
        1570  +
        let first_client = SharedHttpClient::new(
        1571  +
            PoolClient::from_partition(&pool, first_id).expect("first partition should resolve"),
        1572  +
        );
        1573  +
        let second_client = SharedHttpClient::new(
        1574  +
            PoolClient::from_partition(&pool, second_id).expect("second partition should resolve"),
        1575  +
        );
        1576  +
        let first = test_client::connector(&first_client);
        1577  +
        let second = test_client::connector(&second_client);
        1578  +
        1579  +
        let first_request = first.clone();
        1580  +
        let first_url = server.url("/first");
        1581  +
        let (status, body) = first_runtime
        1582  +
            .spawn(async move { test_client::get_and_collect(&first_request, &first_url).await })
        1583  +
            .await
        1584  +
            .expect("first partition request task should not panic");
        1585  +
        assert_eq!((status, body.as_slice()), (200, b"first".as_slice()));
        1586  +
        let original_server_connection = single_stream_connection(&server, "/first");
        1587  +
        let original_open = wait_for_event(&events, |event| {
        1588  +
            event.kind == EventKind::Opened && event.partition == first_id
        1589  +
        })
        1590  +
        .await;
        1591  +
        assert_eq!(Some(first_runtime_id), original_open.runtime);
        1592  +
        assert!(first_runtime.submitted_tasks() > 0);
        1593  +
        1594  +
        let peer_request = second.clone();
        1595  +
        let peer_url = server.url("/peer");
        1596  +
        let peer_response = second_runtime
        1597  +
            .spawn(async move {
        1598  +
                test_client::send_request(
        1599  +
                    &peer_request,
        1600  +
                    HttpRequest::get(peer_url).expect("valid HTTP request"),
        1601  +
                )
        1602  +
                .await
        1603  +
            })
        1604  +
            .await
        1605  +
            .expect("peer request task should not panic")
        1606  +
            .expect("peer response headers should arrive");
        1607  +
        peer_body_gate
        1608  +
            .wait_until_reached(test_client::WAIT)
        1609  +
            .await
        1610  +
            .expect("peer response should reach its body gate");
        1611  +
        assert_eq!(
        1612  +
            original_server_connection,
        1613  +
            single_stream_connection(&server, "/peer"),
        1614  +
            "peer demand should use the first partition's generation"
        1615  +
        );
        1616  +
        assert!(
        1617  +
            !events
        1618  +
                .lock()
        1619  +
                .unwrap()
        1620  +
                .iter()
        1621  +
                .any(|event| { event.kind == EventKind::Opened && event.partition == second_id }),
        1622  +
            "peer reuse should not establish a requester-owned connection"
        1623  +
        );
        1624  +
        1625  +
        first_runtime.shutdown();
        1626  +
        let body_result =
        1627  +
            tokio::time::timeout(test_client::WAIT, peer_response.into_body().collect())
        1628  +
                .await
        1629  +
                .expect("owner-runtime shutdown should terminate the accepted response");
        1630  +
        assert!(
        1631  +
            body_result.is_err(),
        1632  +
            "accepted response completed after its owner runtime stopped"
        1633  +
        );
        1634  +
        let logical_close = wait_for_event(&events, |event| {
        1635  +
            event.kind == EventKind::LogicalClose && event.connection == original_open.connection
        1636  +
        })
        1637  +
        .await;
        1638  +
        assert_eq!(first_id, logical_close.partition);
        1639  +
        let physical_close = wait_for_event(&events, |event| {
        1640  +
            event.kind == EventKind::PhysicalClose && event.connection == original_open.connection
        1641  +
        })
        1642  +
        .await;
        1643  +
        assert_eq!(first_id, physical_close.partition);
        1644  +
        1645  +
        let replacement_request = second.clone();
        1646  +
        let replacement_url = server.url("/replacement");
        1647  +
        let (status, body) = second_runtime
        1648  +
            .spawn(async move {
        1649  +
                test_client::get_and_collect(&replacement_request, &replacement_url).await
        1650  +
            })
        1651  +
            .await
        1652  +
            .expect("replacement request task should not panic");
        1653  +
        assert_eq!((status, body.as_slice()), (200, b"replacement".as_slice()));
        1654  +
        assert_ne!(
        1655  +
            original_server_connection,
        1656  +
            single_stream_connection(&server, "/replacement")
        1657  +
        );
        1658  +
        let replacement_open = wait_for_event(&events, |event| {
        1659  +
            event.kind == EventKind::Opened && event.partition == second_id
        1660  +
        })
        1661  +
        .await;
        1662  +
        assert_eq!(Some(second_runtime_id), replacement_open.runtime);
        1663  +
        assert_ne!(original_open.connection, replacement_open.connection);
        1664  +
        assert!(second_runtime.submitted_tasks() > 0);
        1665  +
        1666  +
        let origin = aws_smithy_http_client::pool::OriginKey::from_uri(
        1667  +
            &server.url("/").parse().expect("valid server URI"),
        1668  +
        )
        1669  +
        .expect("server URI should name an origin");
        1670  +
        let first_stats = pool
        1671  +
            .partition_stats(first_id, &origin)
        1672  +
            .expect("first partition should retain origin statistics");
        1673  +
        assert_eq!(0, first_stats.h2().active_requests());
        1674  +
        assert_eq!(0, first_stats.physically_live_connections());
        1675  +
        let second_stats = pool
        1676  +
            .partition_stats(second_id, &origin)
        1677  +
            .expect("second partition should retain origin statistics");
        1678  +
        assert_eq!(1, second_stats.h2().accepting());
        1679  +
        assert_eq!(0, second_stats.h2().active_requests());
        1680  +
        assert_eq!(1, second_stats.physically_live_connections());
        1681  +
        let capacity = pool
        1682  +
            .origin_stats(&origin)
        1683  +
            .capacity()
        1684  +
            .copied()
        1685  +
            .expect("origin should have bounded capacity");
        1686  +
        assert_eq!(1, capacity.limit());
        1687  +
        assert_eq!(1, capacity.in_use());
        1688  +
        1689  +
        drop(first);
        1690  +
        drop(second);
        1691  +
        drop(first_client);
        1692  +
        drop(second_client);
        1693  +
        drop(pool);
        1694  +
        second_runtime.shutdown();
        1695  +
        server.shutdown().await.expect("clean H2 server shutdown");
        1696  +
    }
        1697  +
}