AWS SDK

AWS SDK

rev. 39e8f0a4a4cd7fc94cae1e2f796e9c21897d9cb9 (ignoring whitespace)

Files changed:

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/client/pool/maintenance.rs

@@ -39,39 +104,98 @@
   59     59   
    }
   60     60   
   61     61   
    /// Returns the deadline for a sender becoming idle now.
   62     62   
    pub(super) fn idle_deadline(&self) -> Option<SystemTime> {
   63     63   
        self.idle_timeout
   64     64   
            .and_then(|timeout| self.time_source.now().checked_add(timeout))
   65     65   
    }
   66     66   
   67     67   
    /// Registers a retained cell without waking an idle task.
   68     68   
    pub(super) fn register(&self, cell: &Arc<OriginCell>) {
   69         -
        // Loom has no modeled Weak. Skip this production-only index to avoid a
   70         -
        // model-only maintenance -> cell -> maintenance cycle.
   71         -
        #[cfg(all(test, smithy_http_client_loom))]
   72         -
        let _ = cell;
   73         -
   74         -
        #[cfg(not(all(test, smithy_http_client_loom)))]
   75     69   
        self.state
   76     70   
            .lock()
   77     71   
            .cells
   78     72   
            .insert(cell.id().clone(), Weak::from_arc(cell));
   79     73   
    }
   80     74   
   81     75   
    /// Publishes newly idle work when it precedes the current wakeup.
   82     76   
    pub(super) fn notify_deadline(&self, deadline: Option<SystemTime>) {
   83     77   
        let Some(deadline) = deadline else {
   84     78   
            return;
@@ -164,158 +223,225 @@
  184    178   
        if state
  185    179   
            .waker
  186    180   
            .as_ref()
  187    181   
            .is_none_or(|registered| !registered.will_wake(cx.waker()))
  188    182   
        {
  189    183   
            state.waker = Some(cx.waker().clone());
  190    184   
        }
  191    185   
        Poll::Pending
  192    186   
    }
  193    187   
         188  +
    #[cfg(all(test, smithy_http_client_loom))]
         189  +
    pub(super) fn clear_modeled_cells_for_test(&self) {
         190  +
        // Loom has no modeled Weak, so its synchronization facade retains
         191  +
        // registered cells strongly. Explicit teardown prevents that model-only
         192  +
        // substitution from creating a maintenance -> cell -> maintenance cycle.
         193  +
        self.state.lock().cells.clear();
         194  +
    }
         195  +
  194    196   
    #[cfg(all(test, feature = "rt-tokio", not(smithy_http_client_loom)))]
  195    197   
    fn probe(&self) -> MaintenanceProbe {
  196    198   
        let state = self.state.lock();
  197    199   
        MaintenanceProbe {
  198    200   
            started: self.started.load(Ordering::Acquire),
  199    201   
            shutdown: state.shutdown,
  200    202   
            scheduled_deadline: state.scheduled_deadline,
  201    203   
        }
  202    204   
    }
  203    205   
}
@@ -360,362 +511,599 @@
  380    382   
#[cfg(all(test, feature = "rt-tokio", not(smithy_http_client_loom)))]
  381    383   
struct MaintenanceProbe {
  382    384   
    started: bool,
  383    385   
    shutdown: bool,
  384    386   
    scheduled_deadline: Option<SystemTime>,
  385    387   
}
  386    388   
  387    389   
#[cfg(all(test, smithy_http_client_loom))]
  388    390   
mod loom_tests {
  389    391   
    use super::*;
  390         -
    use std::sync::atomic::{AtomicUsize, Ordering as StdOrdering};
         392  +
    use crate::client::pool::cell::h1::{H1CloseHandle, H1Sender};
         393  +
    use crate::client::pool::connection::{CloseReason, ConnectionInfo, ConnectionState};
         394  +
    use crate::client::pool::origin::OriginKey;
         395  +
    use crate::client::pool::partition::{EligibilityGroup, PartitionId};
         396  +
    use crate::sync::AtomicUsize;
         397  +
    use aws_smithy_async::test_util::ManualTimeSource;
         398  +
    use aws_smithy_runtime_api::client::connection::ConnectionId;
         399  +
    use http_1x::uri::Scheme;
  391    400   
    use std::sync::Arc as StdArc;
  392    401   
    use std::task::{Context, Wake, Waker};
  393    402   
    use std::time::UNIX_EPOCH;
  394    403   
  395    404   
    struct WakeCounter(AtomicUsize);
  396    405   
  397    406   
    impl Wake for WakeCounter {
  398    407   
        fn wake(self: StdArc<Self>) {
  399         -
            self.0.fetch_add(1, StdOrdering::SeqCst);
         408  +
            self.0.fetch_add(1, Ordering::SeqCst);
  400    409   
        }
  401    410   
  402    411   
        fn wake_by_ref(self: &StdArc<Self>) {
  403         -
            self.0.fetch_add(1, StdOrdering::SeqCst);
         412  +
            self.0.fetch_add(1, Ordering::SeqCst);
  404    413   
        }
  405    414   
    }
  406    415   
         416  +
    /// Races deadline publication with registration of the maintenance waker.
         417  +
    ///
         418  +
    /// A task that observes the old revision must either retry immediately or
         419  +
    /// receive exactly one wake after it parks.
  407    420   
    #[test]
  408    421   
    fn deadline_publication_wakes_a_registered_task() {
  409    422   
        loom::model(|| {
  410    423   
            let maintenance = PartitionMaintenance::new(MaintenanceConfig::default());
  411    424   
            let observed = maintenance
  412    425   
                .begin_scan()
  413    426   
                .expect("new maintenance scheduler was shut down");
  414    427   
            let counter = StdArc::new(WakeCounter(AtomicUsize::new(0)));
  415    428   
            let waker = Waker::from(counter.clone());
  416    429   
            let mut context = Context::from_waker(&waker);
  417    430   
            let publisher = maintenance.clone();
  418    431   
            let publish = loom::thread::spawn(move || {
  419    432   
                publisher.notify_deadline(Some(UNIX_EPOCH + Duration::from_secs(1)));
  420    433   
            });
  421    434   
  422    435   
            let before_join = maintenance.poll_revision(observed, &mut context);
  423    436   
            publish.join().unwrap();
  424    437   
            if before_join.is_pending() {
  425         -
                assert_eq!(1, counter.0.load(StdOrdering::SeqCst));
         438  +
                assert_eq!(1, counter.0.load(Ordering::SeqCst));
  426    439   
            }
  427    440   
            assert!(maintenance.poll_revision(observed, &mut context).is_ready());
  428    441   
        });
  429    442   
    }
  430    443   
         444  +
    /// Publishes an earlier maintenance deadline while a scan is being committed.
         445  +
    ///
         446  +
    /// The scan must retry or retain a revision that forces the next poll to
         447  +
    /// observe the concurrent publication.
  431    448   
    #[test]
  432    449   
    fn deadline_published_during_a_scan_forces_retry_or_wake() {
  433    450   
        loom::model(|| {
  434    451   
            let maintenance = PartitionMaintenance::new(MaintenanceConfig::default());
  435    452   
            let elapsed = UNIX_EPOCH + Duration::from_secs(1);
  436    453   
            let later = UNIX_EPOCH + Duration::from_secs(2);
  437    454   
            maintenance.notify_deadline(Some(elapsed));
  438    455   
            let observed = maintenance
  439    456   
                .begin_scan()
  440    457   
                .expect("new maintenance scheduler was shut down");
  441    458   
            assert_eq!(None, maintenance.state.lock().scheduled_deadline);
  442    459   
  443    460   
            let publisher = maintenance.clone();
  444    461   
            let publish = loom::thread::spawn(move || publisher.notify_deadline(Some(later)));
  445    462   
            let result = maintenance.schedule(observed, None);
  446    463   
            publish.join().unwrap();
  447    464   
  448    465   
            match result {
  449    466   
                ScheduleResult::Retry => {}
  450    467   
                ScheduleResult::Wait { revision, deadline } => {
  451    468   
                    assert_eq!(None, deadline);
  452    469   
                    let state = maintenance.state.lock();
  453    470   
                    assert_ne!(revision, state.revision);
  454    471   
                    assert_eq!(Some(later), state.scheduled_deadline);
  455    472   
                }
  456    473   
                ScheduleResult::Shutdown => panic!("maintenance unexpectedly shut down"),
  457    474   
            }
  458    475   
        });
  459    476   
    }
  460    477   
         478  +
    /// Races shutdown with registration of the maintenance waker.
         479  +
    ///
         480  +
    /// A task that parks before shutdown must be woken and all later polls must
         481  +
    /// observe terminal shutdown.
  461    482   
    #[test]
  462    483   
    fn shutdown_wakes_a_registered_task() {
  463    484   
        loom::model(|| {
  464    485   
            let maintenance = PartitionMaintenance::new(MaintenanceConfig::default());
  465    486   
            let observed = maintenance
  466    487   
                .begin_scan()
  467    488   
                .expect("new maintenance scheduler was shut down");
  468    489   
            let counter = StdArc::new(WakeCounter(AtomicUsize::new(0)));
  469    490   
            let waker = Waker::from(counter.clone());
  470    491   
            let mut context = Context::from_waker(&waker);
  471    492   
            let shutdown = maintenance.clone();
  472    493   
            let stop = loom::thread::spawn(move || shutdown.shutdown());
  473    494   
  474    495   
            let before_join = maintenance.poll_revision(observed, &mut context);
  475    496   
            stop.join().unwrap();
  476    497   
            if before_join.is_pending() {
  477         -
                assert_eq!(1, counter.0.load(StdOrdering::SeqCst));
         498  +
                assert_eq!(1, counter.0.load(Ordering::SeqCst));
  478    499   
            }
  479    500   
            assert!(maintenance.poll_revision(observed, &mut context).is_ready());
  480    501   
        });
  481    502   
    }
         503  +
         504  +
    /// Races an expired idle HTTP/1 record with local sender selection.
         505  +
    ///
         506  +
    /// Selection either takes the sender before the scan observes it or expiry
         507  +
    /// closes the exact record. The sender and record must have one final owner.
         508  +
    #[test]
         509  +
    fn idle_expiry_linearizes_against_h1_selection() {
         510  +
        loom::model(|| {
         511  +
            let maintenance = PartitionMaintenance::new(MaintenanceConfig {
         512  +
                idle_timeout: Some(Duration::ZERO),
         513  +
                time_source: SharedTimeSource::new(ManualTimeSource::new(UNIX_EPOCH)),
         514  +
                sleep: None,
         515  +
            });
         516  +
            let cell = Arc::new(OriginCell::new(
         517  +
                PartitionId::from_index(1),
         518  +
                OriginKey::from_parts(Scheme::HTTP, "example.com", None).unwrap(),
         519  +
                EligibilityGroup::Pool,
         520  +
                None,
         521  +
                Some(maintenance.clone()),
         522  +
            ));
         523  +
            maintenance.register(&cell);
         524  +
            let (connection, physical) = ConnectionState::unbounded(ConnectionInfo::for_test(
         525  +
                ConnectionId::new(1),
         526  +
                cell.id().partition(),
         527  +
            ));
         528  +
            OriginCell::insert_idle_h1(&cell, connection.clone(), H1Sender::test(1));
         529  +
         530  +
            let selecting_cell = cell.clone();
         531  +
            let selecting = loom::thread::spawn(move || OriginCell::select_h1(&selecting_cell));
         532  +
            let scanning = maintenance.clone();
         533  +
            let expiring = loom::thread::spawn(move || {
         534  +
                let observed = scanning
         535  +
                    .begin_scan()
         536  +
                    .expect("maintenance shut down before its scan");
         537  +
                let cells = scanning.cells();
         538  +
                for cell in &cells {
         539  +
                    OriginCell::expire_idle(cell, UNIX_EPOCH);
         540  +
                }
         541  +
                let nearest = cells
         542  +
                    .iter()
         543  +
                    .filter_map(|cell| cell.nearest_idle_deadline())
         544  +
                    .min();
         545  +
                drop(cells);
         546  +
                scanning.schedule(observed, nearest)
         547  +
            });
         548  +
         549  +
            let selected = selecting.join().unwrap();
         550  +
            assert!(matches!(
         551  +
                expiring.join().unwrap(),
         552  +
                ScheduleResult::Wait { .. }
         553  +
            ));
         554  +
            drop(selected);
         555  +
         556  +
            if connection.probe().close_reason.is_none() {
         557  +
                assert!(H1CloseHandle::new(&cell, &connection).close(CloseReason::PoolDropped));
         558  +
            }
         559  +
            let stats = cell.connection_stats_snapshot();
         560  +
            assert_eq!(0, stats.h1().idle());
         561  +
            assert_eq!(0, stats.h1().active());
         562  +
            assert!(matches!(
         563  +
                connection.probe().close_reason,
         564  +
                Some(CloseReason::IdleTimeout | CloseReason::PoolDropped)
         565  +
            ));
         566  +
            drop(physical);
         567  +
            maintenance.clear_modeled_cells_for_test();
         568  +
        });
         569  +
    }
  482    570   
}
  483    571   
  484    572   
#[cfg(all(test, feature = "rt-tokio", not(smithy_http_client_loom)))]
  485    573   
mod tests {
  486    574   
    use super::*;
  487    575   
    use crate::client::pool::cell::h1::H1Sender;
  488    576   
    use crate::client::pool::connection::{ConnectionInfo, ConnectionProtocol, ConnectionState};
  489    577   
    use crate::client::pool::origin::OriginKey;
  490    578   
    use crate::client::pool::partition::{EligibilityGroup, PartitionId, TokioDriverSpawner};
  491    579   
    use aws_smithy_async::test_util::controlled_time_and_sleep;

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/client/pool/registry.rs

@@ -797,797 +856,860 @@
  817    817   
        let registry = explicit_registry([partition(1)], ConnectionReuseScope::Pool, None).unwrap();
  818    818   
        let partition = explicit_partition(&registry, 1);
  819    819   
        let cell = registry
  820    820   
            .resolve_cell(&partition, &"https://example.com/".parse().unwrap())
  821    821   
            .unwrap();
  822    822   
  823    823   
        assert!(cell.admission().is_none());
  824    824   
        assert!(registry.admissions.lock().is_empty());
  825    825   
    }
  826    826   
         827  +
    /// Resolves the same first-use origin concurrently through one partition.
         828  +
    ///
         829  +
    /// Publication must retain one production `OriginCell` instance and return
         830  +
    /// that exact instance to both callers.
  827    831   
    #[test]
  828    832   
    fn first_cell_publication_is_stable_under_contention() {
  829    833   
        const THREADS: usize = 8;
  830    834   
        let registry = Arc::new(
  831    835   
            explicit_registry(
  832    836   
                [partition(1)],
  833    837   
                ConnectionReuseScope::default(),
  834    838   
                NonZeroUsize::new(1),
  835    839   
            )
  836    840   
            .unwrap(),
@@ -870,874 +934,944 @@
  890    894   
        let admission_policy =
  891    895   
            max_connections_per_host.map(|limit| AdmissionPolicy::new(limit, true));
  892    896   
        PartitionRegistry::new(
  893    897   
            Some(partitions.into_iter().collect()),
  894    898   
            reuse_scope,
  895    899   
            admission_policy,
  896    900   
            MaintenanceConfig::default(),
  897    901   
        )
  898    902   
    }
  899    903   
         904  +
    /// Resolves the same first-use origin through two concurrent production calls.
         905  +
    ///
         906  +
    /// Loom enumerates publication schedules while `PartitionRegistry` and
         907  +
    /// `PartitionState` perform their ordinary lookup and insertion transitions.
  900    908   
    #[test]
  901    909   
    fn first_cell_publication_is_stable_under_contention() {
  902    910   
        loom::model(|| {
  903    911   
            let registry = Arc::new(
  904    912   
                explicit_registry(
  905    913   
                    [Partition::new(PartitionId::from_index(1), TestSpawner)],
  906    914   
                    ConnectionReuseScope::default(),
  907    915   
                    None,
  908    916   
                )
  909    917   
                .unwrap(),
  910    918   
            );
  911    919   
            let partition = registry.partition(PartitionId::from_index(1)).unwrap();
  912    920   
  913    921   
            let first_registry = registry.clone();
  914    922   
            let first_partition = partition.clone();
  915    923   
            let first = loom::thread::spawn(move || {
  916    924   
                let uri: Uri = "https://example.com/".parse().unwrap();
  917    925   
                first_registry.resolve_cell(&first_partition, &uri).unwrap()
  918    926   
            });
  919    927   
            let second_registry = registry.clone();
  920    928   
            let second_partition = partition.clone();
  921    929   
            let second = loom::thread::spawn(move || {
  922    930   
                let uri: Uri = "https://example.com/".parse().unwrap();
  923    931   
                second_registry
  924    932   
                    .resolve_cell(&second_partition, &uri)
  925    933   
                    .unwrap()
  926    934   
            });
  927    935   
  928    936   
            let first = first.join().unwrap();
  929    937   
            let second = second.join().unwrap();
  930    938   
            assert!(Arc::ptr_eq(&first, &second));
  931    939   
            assert_eq!(1, partition.cell_count());
         940  +
            drop((first, second));
         941  +
            partition.maintenance.clear_modeled_cells_for_test();
  932    942   
        });
  933    943   
    }
  934    944   
}

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/hyper_legacy.rs

@@ -1,1 +38,40 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
           6  +
use crate::client::ConnectorCallTimer;
    6      7   
use crate::hyper_legacy::timeout_middleware::HttpTimeoutError;
    7      8   
use aws_smithy_async::future::timeout::TimedOutError;
    8      9   
use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep, SharedAsyncSleep};
          10  +
use aws_smithy_async::time::SharedTimeSource;
    9     11   
use aws_smithy_runtime_api::box_error::BoxError;
   10     12   
use aws_smithy_runtime_api::client::connection::CaptureSmithyConnection;
   11     13   
use aws_smithy_runtime_api::client::connection::ConnectionMetadata;
   12     14   
use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata;
   13     15   
use aws_smithy_runtime_api::client::http::{
   14     16   
    HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpClient,
   15     17   
    SharedHttpConnector,
   16     18   
};
   17     19   
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
   18     20   
use aws_smithy_runtime_api::client::result::ConnectorError;
@@ -163,165 +405,418 @@
  183    185   
    fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
  184    186   
        self.adapter.call(request)
  185    187   
    }
  186    188   
}
  187    189   
  188    190   
/// Builder for [`HyperConnector`].
  189    191   
#[derive(Default, Debug)]
  190    192   
pub struct HyperConnectorBuilder {
  191    193   
    connector_settings: Option<HttpConnectorSettings>,
  192    194   
    sleep_impl: Option<SharedAsyncSleep>,
         195  +
    time_source: SharedTimeSource,
  193    196   
    client_builder: Option<hyper_0_14::client::Builder>,
  194    197   
}
  195    198   
  196    199   
impl HyperConnectorBuilder {
  197    200   
    /// Create a [`HyperConnector`] from this builder and a given connector.
  198    201   
    pub fn build<C>(self, tcp_connector: C) -> HyperConnector
  199    202   
    where
  200    203   
        C: Clone + Send + Sync + 'static,
  201    204   
        C: hyper_0_14::service::Service<http_02x::Uri>,
  202    205   
        C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static,
  203    206   
        C::Future: Unpin + Send + 'static,
  204    207   
        C::Error: Into<BoxError>,
  205    208   
    {
  206    209   
        let client_builder = self.client_builder.unwrap_or_default();
  207    210   
        let sleep_impl = self.sleep_impl.or_else(default_async_sleep);
  208    211   
        let (connect_timeout, read_timeout) = self
  209    212   
            .connector_settings
  210    213   
            .map(|c| (c.connect_timeout(), c.read_timeout()))
  211    214   
            .unwrap_or((None, None));
  212    215   
  213    216   
        let connector = match connect_timeout {
  214    217   
            Some(duration) => timeout_middleware::ConnectTimeout::new(
  215    218   
                tcp_connector,
  216    219   
                sleep_impl
  217    220   
                    .clone()
  218    221   
                    .expect("a sleep impl must be provided in order to have a connect timeout"),
  219    222   
                duration,
  220    223   
            ),
  221    224   
            None => timeout_middleware::ConnectTimeout::no_timeout(tcp_connector),
  222    225   
        };
  223    226   
        let base = client_builder.build(connector);
  224    227   
        let read_timeout = match read_timeout {
  225    228   
            Some(duration) => timeout_middleware::HttpReadTimeout::new(
  226    229   
                base,
  227    230   
                sleep_impl.expect("a sleep impl must be provided in order to have a read timeout"),
  228    231   
                duration,
  229    232   
            ),
  230    233   
            None => timeout_middleware::HttpReadTimeout::no_timeout(base),
  231    234   
        };
  232    235   
        HyperConnector {
  233    236   
            adapter: Box::new(Adapter {
  234    237   
                client: read_timeout,
         238  +
                time_source: self.time_source,
  235    239   
            }),
  236    240   
        }
  237    241   
    }
  238    242   
  239    243   
    /// Create a [`HyperConnector`] with the default rustls HTTPS implementation.
  240    244   
    #[cfg(feature = "legacy-rustls-ring")]
  241    245   
    pub fn build_https(self) -> HyperConnector {
  242    246   
        self.build(default_connector::https())
  243    247   
    }
  244    248   
  245    249   
    /// Set the async sleep implementation used for timeouts
  246    250   
    ///
  247    251   
    /// Calling this is only necessary for testing or to use something other than
  248    252   
    /// [`default_async_sleep`].
  249    253   
    pub fn sleep_impl(mut self, sleep_impl: impl AsyncSleep + 'static) -> Self {
  250    254   
        self.sleep_impl = Some(sleep_impl.into_shared());
  251    255   
        self
  252    256   
    }
  253    257   
  254    258   
    /// Set the async sleep implementation used for timeouts
  255    259   
    ///
  256    260   
    /// Calling this is only necessary for testing or to use something other than
  257    261   
    /// [`default_async_sleep`].
  258    262   
    pub fn set_sleep_impl(&mut self, sleep_impl: Option<SharedAsyncSleep>) -> &mut Self {
  259    263   
        self.sleep_impl = sleep_impl;
  260    264   
        self
  261    265   
    }
  262    266   
         267  +
    /// Copies the runtime-component clock used for request telemetry.
         268  +
    fn set_time_source(&mut self, time_source: SharedTimeSource) -> &mut Self {
         269  +
        self.time_source = time_source;
         270  +
        self
         271  +
    }
         272  +
  263    273   
    /// Configure the HTTP settings for the `HyperAdapter`
  264    274   
    pub fn connector_settings(mut self, connector_settings: HttpConnectorSettings) -> Self {
  265    275   
        self.connector_settings = Some(connector_settings);
  266    276   
        self
  267    277   
    }
  268    278   
  269    279   
    /// Configure the HTTP settings for the `HyperAdapter`
  270    280   
    pub fn set_connector_settings(
  271    281   
        &mut self,
  272    282   
        connector_settings: Option<HttpConnectorSettings>,
  273    283   
    ) -> &mut Self {
  274    284   
        self.connector_settings = connector_settings;
  275    285   
        self
  276    286   
    }
  277    287   
  278    288   
    /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client.
  279    289   
    ///
  280    290   
    /// This enables changing settings like forcing HTTP2 and modifying other default client behavior.
  281    291   
    pub fn hyper_builder(mut self, hyper_builder: hyper_0_14::client::Builder) -> Self {
  282    292   
        self.client_builder = Some(hyper_builder);
  283    293   
        self
  284    294   
    }
  285    295   
  286    296   
    /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client.
  287    297   
    ///
  288    298   
    /// This enables changing settings like forcing HTTP2 and modifying other default client behavior.
  289    299   
    pub fn set_hyper_builder(
  290    300   
        &mut self,
  291    301   
        hyper_builder: Option<hyper_0_14::client::Builder>,
  292    302   
    ) -> &mut Self {
  293    303   
        self.client_builder = hyper_builder;
  294    304   
        self
  295    305   
    }
  296    306   
}
  297    307   
  298    308   
/// Adapter from a [`hyper_0_14::Client`] to [`HttpConnector`].
  299    309   
///
  300    310   
/// This adapter also enables TCP `CONNECT` and HTTP `READ` timeouts via [`HyperConnector::builder`].
  301    311   
struct Adapter<C> {
  302    312   
    client: timeout_middleware::HttpReadTimeout<
  303    313   
        hyper_0_14::Client<timeout_middleware::ConnectTimeout<C>, SdkBody>,
  304    314   
    >,
         315  +
    time_source: SharedTimeSource,
  305    316   
}
  306    317   
  307    318   
impl<C> fmt::Debug for Adapter<C> {
  308    319   
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  309    320   
        f.debug_struct("Adapter")
  310    321   
            .field("client", &"** hyper client **")
  311    322   
            .finish()
  312    323   
    }
  313    324   
}
  314    325   
  315    326   
/// Extract a smithy connection from a hyper CaptureConnection
  316    327   
fn extract_smithy_connection(capture_conn: &CaptureConnection) -> Option<ConnectionMetadata> {
  317    328   
    let capture_conn = capture_conn.clone();
  318    329   
    if let Some(conn) = capture_conn.clone().connection_metadata().as_ref() {
  319    330   
        let mut extensions = http_02x::Extensions::new();
  320    331   
        conn.get_extras(&mut extensions);
  321    332   
        let http_info = extensions.get::<HttpInfo>();
  322    333   
        let mut builder = ConnectionMetadata::builder()
  323    334   
            .proxied(conn.is_proxied())
  324    335   
            .poison_fn(move || match capture_conn.connection_metadata().as_ref() {
  325    336   
                Some(conn) => conn.poison(),
  326    337   
                None => tracing::trace!("no connection existed to poison"),
  327    338   
            });
  328    339   
  329    340   
        builder
  330    341   
            .set_local_addr(http_info.map(|info| info.local_addr()))
  331    342   
            .set_remote_addr(http_info.map(|info| info.remote_addr()));
  332    343   
  333    344   
        let smithy_connection = builder.build();
  334    345   
  335    346   
        Some(smithy_connection)
  336    347   
    } else {
  337    348   
        None
  338    349   
    }
  339    350   
}
  340    351   
  341    352   
impl<C> HttpConnector for Adapter<C>
  342    353   
where
  343    354   
    C: Clone + Send + Sync + 'static,
  344    355   
    C: hyper_0_14::service::Service<http_02x::Uri>,
  345    356   
    C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static,
  346    357   
    C::Future: Unpin + Send + 'static,
  347    358   
    C::Error: Into<BoxError>,
  348    359   
{
  349    360   
    fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
  350    361   
        use hyper_0_14::service::Service;
  351    362   
         363  +
        let connector_call_timer = ConnectorCallTimer::start(&request, &self.time_source);
  352    364   
        let mut request = match request.try_into_http02x() {
  353    365   
            Ok(request) => request,
  354    366   
            Err(err) => {
  355    367   
                return HttpConnectorFuture::ready(Err(ConnectorError::other(err.into(), None)));
  356    368   
            }
  357    369   
        };
  358    370   
        let capture_connection = capture_connection(&mut request);
  359    371   
        if let Some(capture_smithy_connection) =
  360    372   
            request.extensions().get::<CaptureSmithyConnection>()
  361    373   
        {
  362    374   
            capture_smithy_connection
  363    375   
                .set_connection_retriever(move || extract_smithy_connection(&capture_connection));
  364    376   
        }
  365    377   
        let mut client = self.client.clone();
  366    378   
        let fut = client.call(request);
  367    379   
        HttpConnectorFuture::new(async move {
  368         -
            let response = fut
  369         -
                .await
  370         -
                .map_err(downcast_error)?
  371         -
                .map(SdkBody::from_body_0_4);
  372         -
            match HttpResponse::try_from(response) {
  373         -
                Ok(response) => Ok(response),
  374         -
                Err(err) => Err(ConnectorError::other(err.into(), None)),
         380  +
            let result = match fut.await {
         381  +
                Ok(response) => HttpResponse::try_from(response.map(SdkBody::from_body_0_4))
         382  +
                    .map_err(|err| ConnectorError::other(err.into(), None)),
         383  +
                Err(err) => Err(downcast_error(err)),
         384  +
            };
         385  +
            if let Some(timer) = connector_call_timer {
         386  +
                timer.finish();
  375    387   
            }
         388  +
            result
  376    389   
        })
  377    390   
    }
  378    391   
}
  379    392   
  380    393   
/// Downcast errors coming out of hyper into an appropriate `ConnectorError`
  381    394   
fn downcast_error(err: BoxError) -> ConnectorError {
  382    395   
    // is a `TimedOutError` (from aws_smithy_async::timeout) in the chain? if it is, this is a timeout
  383    396   
    if find_source::<TimedOutError>(err.as_ref()).is_some() {
  384    397   
        return ConnectorError::timeout(err);
  385    398   
    }
@@ -464,477 +523,539 @@
  484    497   
        let key = CacheKey::from(settings);
  485    498   
        let mut connector = self.connector_cache.read().unwrap().get(&key).cloned();
  486    499   
        if connector.is_none() {
  487    500   
            let mut cache = self.connector_cache.write().unwrap();
  488    501   
            // Short-circuit if another thread already wrote a connector to the cache for this key
  489    502   
            if !cache.contains_key(&key) {
  490    503   
                let mut builder = HyperConnector::builder()
  491    504   
                    .hyper_builder(self.client_builder.clone())
  492    505   
                    .connector_settings(settings.clone());
  493    506   
                builder.set_sleep_impl(components.sleep_impl());
         507  +
                if let Some(time_source) = components.time_source() {
         508  +
                    builder.set_time_source(time_source);
         509  +
                }
  494    510   
  495    511   
                let start = components.time_source().map(|ts| ts.now());
  496    512   
                let tcp_connector = (self.tcp_connector_fn)();
  497    513   
                let end = components.time_source().map(|ts| ts.now());
  498    514   
                if let (Some(start), Some(end)) = (start, end) {
  499    515   
                    if let Ok(elapsed) = end.duration_since(start) {
  500    516   
                        tracing::debug!("new TCP connector created in {:?}", elapsed);
  501    517   
                    }
  502    518   
                }
  503    519   
                let connector = SharedHttpConnector::new(builder.build(tcp_connector));

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/sync/loom.rs

@@ -1,1 +75,76 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
    6         -
//! Loom-backed synchronization primitives used by focused model tests.
           6  +
//! Loom-backed synchronization primitives used to run production pool state
           7  +
//! machines under enumerated thread schedules.
    7      8   
    8      9   
use ::std::ops::{Deref, DerefMut};
    9     10   
   10         -
pub(crate) use loom::sync::atomic::{AtomicBool, Ordering};
          11  +
pub(crate) use loom::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
   11     12   
pub(crate) use loom::sync::Arc;
   12     13   
   13     14   
loom::thread_local! {
   14     15   
    static HELD_POOL_LOCKS: ::std::cell::Cell<usize> = ::std::cell::Cell::new(0);
   15     16   
}
   16     17   
   17     18   
struct LockDepth;
   18     19   
   19     20   
impl LockDepth {
   20     21   
    fn enter() -> Self {
   21     22   
        HELD_POOL_LOCKS.with(|depth| {
   22     23   
            assert_eq!(
   23     24   
                0,
   24     25   
                depth.get(),
   25     26   
                "pool coordination locks must never be nested"
   26     27   
            );
   27     28   
            depth.set(1);
   28     29   
        });
   29     30   
        Self
   30     31   
    }
   31     32   
}
   32     33   
   33     34   
impl Drop for LockDepth {
   34     35   
    fn drop(&mut self) {
   35     36   
        HELD_POOL_LOCKS.with(|depth| {
   36     37   
            assert_eq!(1, depth.get(), "pool lock-depth tracking became unbalanced");
   37     38   
            depth.set(0);
   38     39   
        });
   39     40   
    }
   40     41   
}
   41     42   
   42     43   
/// Loom substitute for a non-owning registry reference.
   43     44   
///
   44     45   
/// Loom's modeled `Arc` has no weak-reference API. Keeping a strong reference
   45         -
/// in model tests preserves lookup behavior but cannot model target expiry.
          46  +
/// in Loom tests preserves lookup behavior but cannot model target expiry.
   46     47   
/// Ordinary tests cover the expired-target delivery fallback.
   47     48   
#[derive(Debug)]
   48     49   
pub(crate) struct Weak<T>(Arc<T>);
   49     50   
   50     51   
impl<T> Weak<T> {
   51     52   
    /// Creates the modeled registry reference.
   52     53   
    pub(crate) fn from_arc(value: &Arc<T>) -> Self {
   53     54   
        Self(value.clone())
   54     55   
    }
   55     56   

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/src/test_util/wire/connection.rs

@@ -76,76 +135,137 @@
   96     96   
/// Why a connection task stopped.
   97     97   
#[non_exhaustive]
   98     98   
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
   99     99   
pub enum ConnectionCloseReason {
  100    100   
    /// The peer closed the connection.
  101    101   
    ClientClosed,
  102    102   
    /// The script reached its end or explicitly closed the connection.
  103    103   
    ScriptCompleted,
  104    104   
    /// The script reset the connection.
  105    105   
    Reset,
         106  +
    /// The harness dropped the transport without a protocol close signal.
         107  +
    ScriptedTransportAbort,
  106    108   
    /// The harness was shut down.
  107    109   
    HarnessShutdown,
  108    110   
    /// The script failed.
  109    111   
    ScriptFailed,
  110    112   
}
  111    113   
  112    114   
/// An event recorded by the harness.
  113    115   
#[non_exhaustive]
  114    116   
#[derive(Clone, Debug, Eq, PartialEq)]
  115    117   
pub enum ConnectionEvent {

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/client.rs

@@ -1,1 +39,39 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
    6      6   
//! Client configuration and request helpers shared by connection behavior tests.
    7      7   
    8      8   
use aws_smithy_async::rt::sleep::{SharedAsyncSleep, TokioSleep};
    9         -
use aws_smithy_async::time::SystemTimeSource;
           9  +
use aws_smithy_async::time::{SystemTimeSource, TimeSource};
   10     10   
use aws_smithy_http_client::pool::{Client as PoolClient, ConnectionPool};
   11     11   
use aws_smithy_http_client::proxy::ProxyConfig;
   12     12   
#[cfg(any(feature = "__rustls", feature = "s2n-tls"))]
   13     13   
use aws_smithy_http_client::tls;
   14     14   
use aws_smithy_http_client::{Builder, Connector};
   15     15   
use aws_smithy_runtime_api::client::dns::SharedDnsResolver;
   16     16   
use aws_smithy_runtime_api::client::http::{
   17     17   
    http_client_fn, HttpClient, HttpConnector, HttpConnectorSettings, SharedHttpClient,
   18     18   
    SharedHttpConnector,
   19     19   
};
@@ -219,219 +278,292 @@
  239    239   
    client: &SharedHttpClient,
  240    240   
    settings: HttpConnectorSettings,
  241    241   
) -> SharedHttpConnector {
  242    242   
    client.http_connector(&settings, &runtime_components())
  243    243   
}
  244    244   
  245    245   
pub(crate) fn connector(client: &SharedHttpClient) -> SharedHttpConnector {
  246    246   
    connector_with_settings(client, HttpConnectorSettings::builder().build())
  247    247   
}
  248    248   
         249  +
/// Builds a connector whose request telemetry uses `time_source`.
         250  +
#[allow(dead_code)]
         251  +
pub(crate) fn connector_with_time_source(
         252  +
    client: &SharedHttpClient,
         253  +
    time_source: impl TimeSource + 'static,
         254  +
) -> SharedHttpConnector {
         255  +
    let components = RuntimeComponentsBuilder::for_tests()
         256  +
        .with_time_source(Some(time_source))
         257  +
        .with_sleep_impl(Some(SharedAsyncSleep::new(TokioSleep::new())))
         258  +
        .build()
         259  +
        .expect("valid runtime components");
         260  +
    client.http_connector(&HttpConnectorSettings::builder().build(), &components)
         261  +
}
         262  +
  249    263   
pub(crate) async fn send_request(
  250    264   
    connector: &SharedHttpConnector,
  251    265   
    request: HttpRequest,
  252    266   
) -> Result<HttpResponse, ConnectorError> {
  253    267   
    tokio::time::timeout(WAIT, connector.call(request))
  254    268   
        .await
  255    269   
        .expect("request should finish within the outer deadline")
  256    270   
}
  257    271   
  258    272   
pub(crate) async fn send_and_collect(

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/h2.rs

@@ -170,170 +292,322 @@
  190    190   
  191    191   
    pub(crate) fn body(mut self, body: H2BodyPlan) -> Self {
  192    192   
        self.body = body;
  193    193   
        self
  194    194   
    }
  195    195   
}
  196    196   
  197    197   
#[derive(Clone, Debug)]
  198    198   
pub(crate) enum H2StreamScript {
  199    199   
    Respond(H2Response),
         200  +
    /// Waits at the response-head boundary before sending the response.
         201  +
    RespondAfter {
         202  +
        response: H2Response,
         203  +
        send: GateWaiter,
         204  +
    },
  200    205   
    /// Sends the response before a gate permits the request body to be drained.
  201    206   
    RespondBeforeReceivingRequestBody {
  202    207   
        response: H2Response,
  203    208   
        receive: GateWaiter,
  204    209   
    },
  205    210   
    Reset(Reason),
  206    211   
}
  207    212   
  208    213   
impl H2StreamScript {
  209    214   
    pub(crate) fn respond(response: H2Response) -> Self {
  210    215   
        Self::Respond(response)
  211    216   
    }
  212    217   
         218  +
    /// Delays the response head until the gate is released.
         219  +
    pub(crate) fn respond_after(response: H2Response, send: GateWaiter) -> Self {
         220  +
        Self::RespondAfter { response, send }
         221  +
    }
         222  +
  213    223   
    /// Delays request-body reads until after the response and gate release.
  214    224   
    pub(crate) fn respond_before_receiving_request_body(
  215    225   
        response: H2Response,
  216    226   
        receive: GateWaiter,
  217    227   
    ) -> Self {
  218    228   
        Self::RespondBeforeReceivingRequestBody { response, receive }
  219    229   
    }
  220    230   
  221    231   
    pub(crate) fn reset(reason: Reason) -> Self {
  222    232   
        Self::Reset(reason)
  223    233   
    }
  224    234   
}
  225    235   
  226    236   
#[derive(Clone, Debug, Default)]
  227    237   
pub(crate) struct H2ConnectionScript {
  228    238   
    routes: HashMap<String, H2StreamScript>,
  229    239   
    fallback: Option<H2StreamScript>,
  230    240   
    allow_handshake_abandonment: bool,
  231    241   
    goaway_on_ready: bool,
         242  +
    client_close_behavior: ClientCloseBehavior,
         243  +
}
         244  +
         245  +
/// How a scripted connection handles active stream tasks after client close.
         246  +
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
         247  +
enum ClientCloseBehavior {
         248  +
    /// Wait for stream scripts and report their failures.
         249  +
    #[default]
         250  +
    Drain,
         251  +
    /// Abort scripts that a test deliberately leaves blocked.
         252  +
    AbortStreams,
  232    253   
}
  233    254   
  234    255   
impl H2ConnectionScript {
  235    256   
    pub(crate) fn new() -> Self {
  236    257   
        Self::default()
  237    258   
    }
  238    259   
  239    260   
    pub(crate) fn route(mut self, path: impl Into<String>, script: H2StreamScript) -> Self {
  240    261   
        let path = path.into();
  241    262   
        assert!(
  242    263   
            self.routes.insert(path.clone(), script).is_none(),
  243    264   
            "duplicate H2 route {path:?}"
  244    265   
        );
  245    266   
        self
  246    267   
    }
  247    268   
  248    269   
    pub(crate) fn fallback(mut self, script: H2StreamScript) -> Self {
  249    270   
        self.fallback = Some(script);
  250    271   
        self
  251    272   
    }
  252    273   
  253    274   
    pub(crate) fn allow_handshake_abandonment(mut self) -> Self {
  254    275   
        self.allow_handshake_abandonment = true;
  255    276   
        self
  256    277   
    }
  257    278   
  258    279   
    pub(crate) fn goaway_on_ready(mut self) -> Self {
  259    280   
        self.goaway_on_ready = true;
  260    281   
        self
  261    282   
    }
  262    283   
         284  +
    /// Aborts active stream scripts when the client closes the connection.
         285  +
    ///
         286  +
    /// Use this only when the test intentionally strands a stream while
         287  +
    /// terminating the runtime that owns its transport.
         288  +
    pub(crate) fn abort_streams_on_client_close(mut self) -> Self {
         289  +
        self.client_close_behavior = ClientCloseBehavior::AbortStreams;
         290  +
        self
         291  +
    }
         292  +
  263    293   
    fn script_for(&self, path: &str) -> Option<H2StreamScript> {
  264    294   
        self.routes.get(path).or(self.fallback.as_ref()).cloned()
  265    295   
    }
  266    296   
}
  267    297   
  268    298   
#[derive(Clone, Debug)]
  269    299   
pub(crate) enum H2ConnectionPlan {
  270    300   
    Queue(VecDeque<H2ConnectionScript>),
  271    301   
    Unbounded(H2ConnectionScript),
  272    302   
}
@@ -367,397 +426,457 @@
  387    417   
            .lock()
  388    418   
            .unwrap_or_else(|err| err.into_inner())
  389    419   
            .get(&connection_id)
  390    420   
            .cloned()
  391    421   
    }
  392    422   
}
  393    423   
  394    424   
#[derive(Debug)]
  395    425   
enum ConnectionCommand {
  396    426   
    GracefulShutdown(oneshot::Sender<()>),
         427  +
    AbortTransport(oneshot::Sender<()>),
  397    428   
}
  398    429   
  399    430   
#[derive(Debug, Default)]
  400    431   
pub(crate) struct H2ServerBuilder {
  401    432   
    connections: Option<H2ConnectionPlan>,
  402    433   
}
  403    434   
  404    435   
impl H2ServerBuilder {
  405    436   
    pub(crate) fn connections(mut self, connections: H2ConnectionPlan) -> Self {
  406    437   
        self.connections = Some(connections);
@@ -506,537 +565,624 @@
  526    557   
                    "timed out waiting for H2 connection {connection_id:?} to start graceful shutdown"
  527    558   
                ))
  528    559   
            })?
  529    560   
            .map_err(|_| {
  530    561   
                H2HarnessError::new(format!(
  531    562   
                    "H2 connection {connection_id:?} closed before graceful shutdown started"
  532    563   
                ))
  533    564   
            })
  534    565   
    }
  535    566   
         567  +
    /// Drops one active transport without sending GOAWAY.
         568  +
    pub(crate) async fn abort_transport(
         569  +
        &self,
         570  +
        connection_id: H2ConnectionId,
         571  +
    ) -> Result<(), H2HarnessError> {
         572  +
        let control = self.state.control(connection_id).ok_or_else(|| {
         573  +
            H2HarnessError::new(format!("H2 connection {connection_id:?} is not active"))
         574  +
        })?;
         575  +
        let (acknowledged, ack) = oneshot::channel();
         576  +
        control
         577  +
            .send(ConnectionCommand::AbortTransport(acknowledged))
         578  +
            .map_err(|_| {
         579  +
                H2HarnessError::new(format!("failed to abort H2 transport {connection_id:?}"))
         580  +
            })?;
         581  +
        tokio::time::timeout(WAIT, ack)
         582  +
            .await
         583  +
            .map_err(|_| {
         584  +
                H2HarnessError::new(format!(
         585  +
                    "timed out waiting to abort H2 transport {connection_id:?}"
         586  +
                ))
         587  +
            })?
         588  +
            .map_err(|_| {
         589  +
                H2HarnessError::new(format!(
         590  +
                    "H2 connection {connection_id:?} closed before its transport was aborted"
         591  +
                ))
         592  +
            })
         593  +
    }
         594  +
  536    595   
    pub(crate) async fn shutdown(mut self) -> Result<(), H2HarnessError> {
  537    596   
        self.shutdown.send_replace(true);
  538    597   
        let mut listener_task = self
  539    598   
            .listener_task
  540    599   
            .take()
  541    600   
            .expect("listener task should exist until shutdown");
  542    601   
        match tokio::time::timeout(WAIT, &mut listener_task).await {
  543    602   
            Ok(Ok(())) => {}
  544    603   
            Ok(Err(err)) => {
  545    604   
                return Err(H2HarnessError::new(format!(
@@ -710,769 +790,855 @@
  730    789   
            }
  731    790   
            Ok(Err(err)) => return Err(H2HarnessError::new(format!("H2 handshake failed: {err}"))),
  732    791   
        };
  733    792   
    state.record_event(H2Event::H2Ready { connection_id });
  734    793   
    if script.goaway_on_ready {
  735    794   
        connection.abrupt_shutdown(Reason::NO_ERROR);
  736    795   
    }
  737    796   
  738    797   
    let mut stream_tasks = JoinSet::new();
  739    798   
    let mut shutting_down = false;
         799  +
    let mut transport_aborted = false;
  740    800   
    let mut graceful_shutdown = script.goaway_on_ready;
  741    801   
    let mut control_open = true;
  742    802   
    // Tracks whether the connection ended because the client closed it (accept returned None)
  743    803   
    // vs. a scripted GOAWAY completing.
  744    804   
    let mut client_initiated_close = false;
  745    805   
  746    806   
    loop {
  747    807   
        tokio::select! {
  748    808   
            changed = shutdown.changed(), if !shutting_down => {
  749    809   
                if changed.is_err() || *shutdown.borrow() {
  750    810   
                    shutting_down = true;
  751    811   
                    connection.abrupt_shutdown(Reason::NO_ERROR);
  752    812   
                }
  753    813   
            }
  754    814   
            command = control.recv(), if control_open => {
  755    815   
                match command {
  756    816   
                    Some(ConnectionCommand::GracefulShutdown(acknowledged)) => {
  757    817   
                        graceful_shutdown = true;
  758    818   
                        connection.graceful_shutdown();
  759    819   
                        let _ = acknowledged.send(());
  760    820   
                    }
         821  +
                    Some(ConnectionCommand::AbortTransport(acknowledged)) => {
         822  +
                        transport_aborted = true;
         823  +
                        let _ = acknowledged.send(());
         824  +
                        break;
         825  +
                    }
  761    826   
                    None => control_open = false,
  762    827   
                }
  763    828   
            }
  764    829   
            accepted = connection.accept() => {
  765    830   
                match accepted {
  766    831   
                    Some(Ok((request, mut respond))) => {
  767    832   
                        let stream_id = respond.stream_id().as_u32();
  768    833   
                        let method = request.method().clone();
  769    834   
                        let path = request.uri().path().to_string();
  770    835   
                        state.record_event(H2Event::StreamAccepted {
@@ -802,867 +917,1019 @@
  822    887   
                        break;
  823    888   
                    }
  824    889   
                }
  825    890   
            }
  826    891   
            completed = stream_tasks.join_next(), if !stream_tasks.is_empty() => {
  827    892   
                record_stream_task_result(&state, completed);
  828    893   
            }
  829    894   
        }
  830    895   
    }
  831    896   
  832         -
    if shutting_down {
         897  +
    let abort_streams = shutting_down
         898  +
        || transport_aborted
         899  +
        || (client_initiated_close
         900  +
            && script.client_close_behavior == ClientCloseBehavior::AbortStreams);
         901  +
    if abort_streams {
  833    902   
        stream_tasks.abort_all();
  834    903   
        while stream_tasks.join_next().await.is_some() {}
  835         -
        Ok(ConnectionCloseReason::HarnessShutdown)
         904  +
        Ok(aborted_connection_close_reason(
         905  +
            transport_aborted,
         906  +
            shutting_down,
         907  +
            client_initiated_close,
         908  +
        ))
  836    909   
    } else {
  837    910   
        let drain = async {
  838    911   
            while let Some(completed) = stream_tasks.join_next().await {
  839    912   
                record_stream_task_result(&state, Some(completed));
  840    913   
            }
  841    914   
        };
  842    915   
        if tokio::time::timeout(WAIT, drain).await.is_err() {
  843    916   
            state.record_failure(H2HarnessError::new(format!(
  844    917   
                "H2 connection {connection_id:?} closed before its stream scripts completed"
  845    918   
            )));
  846    919   
            stream_tasks.abort_all();
  847    920   
            while stream_tasks.join_next().await.is_some() {}
  848    921   
        }
  849    922   
        if client_initiated_close {
  850    923   
            Ok(ConnectionCloseReason::ClientClosed)
  851    924   
        } else {
  852    925   
            Ok(ConnectionCloseReason::ScriptCompleted)
  853    926   
        }
  854    927   
    }
  855    928   
}
  856    929   
         930  +
fn aborted_connection_close_reason(
         931  +
    transport_aborted: bool,
         932  +
    shutting_down: bool,
         933  +
    client_initiated_close: bool,
         934  +
) -> ConnectionCloseReason {
         935  +
    debug_assert!(transport_aborted || shutting_down || client_initiated_close);
         936  +
    if transport_aborted {
         937  +
        ConnectionCloseReason::ScriptedTransportAbort
         938  +
    } else if shutting_down {
         939  +
        ConnectionCloseReason::HarnessShutdown
         940  +
    } else {
         941  +
        ConnectionCloseReason::ClientClosed
         942  +
    }
         943  +
}
         944  +
         945  +
#[test]
         946  +
fn harness_shutdown_precedes_client_close() {
         947  +
    assert_eq!(
         948  +
        ConnectionCloseReason::HarnessShutdown,
         949  +
        aborted_connection_close_reason(false, true, true,)
         950  +
    );
         951  +
}
         952  +
  857    953   
fn record_stream_task_result(
  858    954   
    state: &SharedState,
  859    955   
    completed: Option<Result<Result<(), H2HarnessError>, tokio::task::JoinError>>,
  860    956   
) {
  861    957   
    match completed {
  862    958   
        Some(Ok(Ok(()))) | None => {}
  863    959   
        Some(Ok(Err(err))) => state.record_failure(err),
  864    960   
        Some(Err(err)) => {
  865    961   
            state.record_failure(H2HarnessError::new(format!("H2 stream task failed: {err}")))
  866    962   
        }
  867    963   
    }
  868    964   
}
  869    965   
  870    966   
async fn run_stream(
  871    967   
    connection_id: H2ConnectionId,
  872    968   
    stream_id: u32,
  873    969   
    script: H2StreamScript,
  874    970   
    mut respond: SendResponse<Bytes>,
  875    971   
    state: Arc<SharedState>,
  876    972   
    held_request: Option<(RecvStream, GateWaiter)>,
  877    973   
) -> Result<(), H2HarnessError> {
         974  +
    if let H2StreamScript::RespondAfter { send, .. } = &script {
         975  +
        send.wait()
         976  +
            .await
         977  +
            .map_err(|err| H2HarnessError::new(format!("H2 response-head gate failed: {err}")))?;
         978  +
    }
  878    979   
    match script {
  879    980   
        H2StreamScript::Reset(reason) => {
  880    981   
            respond.send_reset(reason);
  881    982   
            state.record_event(H2Event::ResetSent {
  882    983   
                connection_id,
  883    984   
                stream_id,
  884    985   
                reason,
  885    986   
            });
  886    987   
        }
  887    988   
        H2StreamScript::Respond(response)
         989  +
        | H2StreamScript::RespondAfter { response, send: _ }
  888    990   
        | H2StreamScript::RespondBeforeReceivingRequestBody {
  889    991   
            response,
  890    992   
            receive: _,
  891    993   
        } => {
  892    994   
            let end_stream =
  893    995   
                matches!(&response.body, H2BodyPlan::Complete(body) if body.is_empty());
  894    996   
            let response_head = Response::builder()
  895    997   
                .status(response.status)
  896    998   
                .body(())
  897    999   
                .map_err(|err| {

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/common/runtime.rs

@@ -0,1 +0,133 @@
           1  +
/*
           2  +
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
           3  +
 * SPDX-License-Identifier: Apache-2.0
           4  +
 */
           5  +
           6  +
//! Independently driven Tokio runtimes for partition-placement tests.
           7  +
           8  +
use aws_smithy_http_client::pool::DriverSpawner;
           9  +
use std::future::Future;
          10  +
use std::pin::Pin;
          11  +
use std::sync::atomic::{AtomicUsize, Ordering};
          12  +
use std::sync::{mpsc, Arc};
          13  +
use std::thread::{self, JoinHandle};
          14  +
use tokio::runtime::{Handle, Id};
          15  +
use tokio::sync::oneshot;
          16  +
          17  +
/// A current-thread Tokio runtime driven by its own operating-system thread.
          18  +
#[derive(Debug)]
          19  +
pub(crate) struct DrivenRuntime {
          20  +
    handle: Handle,
          21  +
    runtime_id: Id,
          22  +
    shutdown: Option<oneshot::Sender<()>>,
          23  +
    thread: Option<JoinHandle<()>>,
          24  +
    submitted_tasks: Arc<AtomicUsize>,
          25  +
}
          26  +
          27  +
impl DrivenRuntime {
          28  +
    /// Starts a current-thread runtime and waits until its handle is available.
          29  +
    pub(crate) fn start(name: &str) -> Self {
          30  +
        let (ready_tx, ready_rx) = mpsc::sync_channel(1);
          31  +
        let (shutdown, shutdown_rx) = oneshot::channel();
          32  +
        let submitted_tasks = Arc::new(AtomicUsize::new(0));
          33  +
        let thread = thread::Builder::new()
          34  +
            .name(name.to_owned())
          35  +
            .spawn(move || {
          36  +
                let runtime = tokio::runtime::Builder::new_current_thread()
          37  +
                    .enable_all()
          38  +
                    .build()
          39  +
                    .expect("test runtime should build");
          40  +
                let handle = runtime.handle().clone();
          41  +
                let runtime_id = handle.id();
          42  +
                ready_tx
          43  +
                    .send((handle, runtime_id))
          44  +
                    .expect("runtime owner should receive its handle");
          45  +
                runtime.block_on(async {
          46  +
                    let _ = shutdown_rx.await;
          47  +
                });
          48  +
            })
          49  +
            .expect("test runtime thread should start");
          50  +
        let (handle, runtime_id) = ready_rx
          51  +
            .recv()
          52  +
            .expect("test runtime should publish its handle");
          53  +
        Self {
          54  +
            handle,
          55  +
            runtime_id,
          56  +
            shutdown: Some(shutdown),
          57  +
            thread: Some(thread),
          58  +
            submitted_tasks,
          59  +
        }
          60  +
    }
          61  +
          62  +
    /// Returns a spawner that checks task placement on this runtime.
          63  +
    pub(crate) fn driver_spawner(&self) -> RuntimeDriverSpawner {
          64  +
        RuntimeDriverSpawner {
          65  +
            handle: self.handle.clone(),
          66  +
            runtime_id: self.runtime_id,
          67  +
            submitted_tasks: self.submitted_tasks.clone(),
          68  +
        }
          69  +
    }
          70  +
          71  +
    /// Spawns test orchestration work on this runtime.
          72  +
    pub(crate) fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>
          73  +
    where
          74  +
        F: Future + Send + 'static,
          75  +
        F::Output: Send + 'static,
          76  +
    {
          77  +
        self.handle.spawn(future)
          78  +
    }
          79  +
          80  +
    /// Returns the runtime's stable Tokio identity.
          81  +
    pub(crate) fn id(&self) -> Id {
          82  +
        self.runtime_id
          83  +
    }
          84  +
          85  +
    /// Returns how many connection-owned tasks have started on this runtime.
          86  +
    pub(crate) fn submitted_tasks(&self) -> usize {
          87  +
        self.submitted_tasks.load(Ordering::SeqCst)
          88  +
    }
          89  +
          90  +
    /// Stops the runtime, drops all runtime-owned tasks, and joins its thread.
          91  +
    pub(crate) fn shutdown(mut self) {
          92  +
        self.stop();
          93  +
    }
          94  +
          95  +
    fn stop(&mut self) {
          96  +
        if let Some(shutdown) = self.shutdown.take() {
          97  +
            let _ = shutdown.send(());
          98  +
        }
          99  +
        if let Some(thread) = self.thread.take() {
         100  +
            thread.join().expect("test runtime thread should not panic");
         101  +
        }
         102  +
    }
         103  +
}
         104  +
         105  +
impl Drop for DrivenRuntime {
         106  +
    fn drop(&mut self) {
         107  +
        self.stop();
         108  +
    }
         109  +
}
         110  +
         111  +
/// Places connection-owned tasks on one [`DrivenRuntime`].
         112  +
#[derive(Clone, Debug)]
         113  +
pub(crate) struct RuntimeDriverSpawner {
         114  +
    handle: Handle,
         115  +
    runtime_id: Id,
         116  +
    submitted_tasks: Arc<AtomicUsize>,
         117  +
}
         118  +
         119  +
impl DriverSpawner for RuntimeDriverSpawner {
         120  +
    fn spawn(&self, driver: Pin<Box<dyn Future<Output = ()> + Send + 'static>>) {
         121  +
        let runtime_id = self.runtime_id;
         122  +
        let submitted_tasks = self.submitted_tasks.clone();
         123  +
        drop(self.handle.spawn(async move {
         124  +
            assert_eq!(
         125  +
                runtime_id,
         126  +
                Handle::current().id(),
         127  +
                "connection-owned task started on the wrong runtime"
         128  +
            );
         129  +
            submitted_tasks.fetch_add(1, Ordering::SeqCst);
         130  +
            driver.await;
         131  +
        }));
         132  +
    }
         133  +
}

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/connection_pool_test.rs

@@ -1,1 +64,71 @@
    9      9   
    feature = "wire-mock",
   10     10   
    feature = "default-client",
   11     11   
    feature = "rt-tokio"
   12     12   
))]
   13     13   
   14     14   
mod common {
   15     15   
    #[allow(dead_code)]
   16     16   
    pub(crate) mod client;
   17     17   
}
   18     18   
          19  +
#[path = "common/runtime.rs"]
          20  +
mod runtime;
          21  +
          22  +
use aws_smithy_async::test_util::ManualTimeSource;
   19     23   
use aws_smithy_http_client::pool::{
   20     24   
    Client, ConnectPath, ConnectionEstablishmentId, ConnectionEstablishmentStage, ConnectionEvent,
   21     25   
    ConnectionPool, ConnectionProtocol, ConnectionReuseScope, OriginKey, Partition, PartitionId,
   22     26   
    TokioDriverSpawner,
   23     27   
};
   24     28   
use aws_smithy_http_client::test_util::wire::connection::{
   25     29   
    BodyPlan, ConnectionCloseReason, ConnectionEvent as WireConnectionEvent, ConnectionTestHarness,
   26         -
    EndpointPlan, Http1Response, Http1Script, ManualGate,
          30  +
    EndpointPlan, Http1Response, Http1Script, ManualGate, SocketScript,
          31  +
};
          32  +
use aws_smithy_runtime_api::client::connection::{CaptureSmithyConnection, ConnectionId};
          33  +
use aws_smithy_runtime_api::client::http::telemetry::{
          34  +
    CaptureHttpAttemptTelemetry, ConnectionUsage,
   27     35   
};
   28         -
use aws_smithy_runtime_api::client::connection::ConnectionId;
   29     36   
use aws_smithy_runtime_api::client::http::{SharedHttpClient, SharedHttpConnector};
   30     37   
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
   31     38   
use common::client as test_client;
   32     39   
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
   33     40   
use std::sync::{Arc, Mutex};
   34         -
use std::time::Duration;
          41  +
use std::time::{Duration, UNIX_EPOCH};
   35     42   
use tokio::io::{AsyncReadExt, AsyncWriteExt};
   36     43   
use tokio::net::TcpListener;
   37     44   
   38     45   
const IP1: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
   39     46   
const IP2: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2));
   40     47   
   41     48   
#[derive(Debug)]
   42     49   
enum ObservedConnectionEvent {
   43     50   
    Failed {
   44     51   
        establishment: ConnectionEstablishmentId,
@@ -292,299 +407,582 @@
  312    319   
    let (status, body) = test_client::get_and_collect(&connector, &url).await;
  313    320   
    assert_eq!((status, body.as_slice()), (200, b"custom dns".as_slice()));
  314    321   
    assert_eq!(1, harness.dns_lookup_count());
  315    322   
  316    323   
    drop(connector);
  317    324   
    drop(client);
  318    325   
    drop(pool);
  319    326   
    harness.shutdown().await.expect("clean harness shutdown");
  320    327   
}
  321    328   
         329  +
#[tokio::test]
         330  +
async fn h1_attempt_telemetry_distinguishes_fresh_and_reused_connections() {
         331  +
    let harness = ConnectionTestHarness::builder()
         332  +
        .endpoint(
         333  +
            IP1,
         334  +
            Http1Script::responses([
         335  +
                Http1Response::ok().body("first"),
         336  +
                Http1Response::ok().body("second"),
         337  +
            ]),
         338  +
        )
         339  +
        .build()
         340  +
        .await
         341  +
        .expect("harness should start");
         342  +
    let opened = Arc::new(Mutex::new(None));
         343  +
    let pool = ConnectionPool::builder()
         344  +
        .event_listener({
         345  +
            let opened = opened.clone();
         346  +
            move |event: &ConnectionEvent<'_>| {
         347  +
                if let ConnectionEvent::Opened(event) = event {
         348  +
                    *opened.lock().unwrap() =
         349  +
                        Some((event.connection().id(), event.stats().total_duration()));
         350  +
                }
         351  +
            }
         352  +
        })
         353  +
        .build_http()
         354  +
        .expect("valid pool");
         355  +
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
         356  +
    let connector = connector(&client);
         357  +
         358  +
    let first_connection_capture = CaptureSmithyConnection::new();
         359  +
    let first_capture = CaptureHttpAttemptTelemetry::new();
         360  +
    let mut first = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         361  +
    first.add_extension(first_connection_capture.clone());
         362  +
    first.add_extension(first_capture.clone());
         363  +
    test_client::send_and_collect(&connector, first).await;
         364  +
         365  +
    let second_capture = CaptureHttpAttemptTelemetry::new();
         366  +
    let mut second = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         367  +
    second.add_extension(second_capture.clone());
         368  +
    test_client::send_and_collect(&connector, second).await;
         369  +
         370  +
    let first = first_capture.get();
         371  +
    let second = second_capture.get();
         372  +
    assert!(first.connector_call_duration().is_some());
         373  +
    assert!(second.connector_call_duration().is_some());
         374  +
    assert_eq!(
         375  +
        first.acquisition().expect("first acquisition").usage(),
         376  +
        ConnectionUsage::Fresh
         377  +
    );
         378  +
    assert_eq!(
         379  +
        second.acquisition().expect("second acquisition").usage(),
         380  +
        ConnectionUsage::Reused
         381  +
    );
         382  +
    let first_connection = first.connection().expect("first connection metadata");
         383  +
    let second_connection = second.connection().expect("second connection metadata");
         384  +
    let captured_connection = first_connection_capture
         385  +
        .get()
         386  +
        .expect("connection capture should contain selected metadata");
         387  +
    assert_eq!(
         388  +
        captured_connection.connection_id(),
         389  +
        first_connection.connection_id()
         390  +
    );
         391  +
    assert_eq!(
         392  +
        captured_connection.establishment(),
         393  +
        first_connection.establishment(),
         394  +
        "connection and attempt captures must observe the same selected connection"
         395  +
    );
         396  +
    assert_eq!(
         397  +
        first_connection.connection_id(),
         398  +
        second_connection.connection_id()
         399  +
    );
         400  +
    assert_eq!(
         401  +
        first_connection.establishment(),
         402  +
        second_connection.establishment(),
         403  +
        "reuse must retain the establishment that created the connection"
         404  +
    );
         405  +
    let first_establishment = first_connection
         406  +
        .establishment()
         407  +
        .expect("first connection establishment");
         408  +
    let (opened_connection, opened_duration) =
         409  +
        opened.lock().unwrap().expect("opened connection event");
         410  +
    assert_eq!(
         411  +
        first_connection.connection_id(),
         412  +
        Some(opened_connection),
         413  +
        "request telemetry and lifecycle events must identify the same connection"
         414  +
    );
         415  +
    assert_eq!(
         416  +
        first_establishment.total_duration(),
         417  +
        opened_duration,
         418  +
        "request metadata and the opened event must share one establishment measurement"
         419  +
    );
         420  +
    assert_eq!(1, harness.tcp_accepted_count());
         421  +
         422  +
    drop(connector);
         423  +
    drop(client);
         424  +
    drop(pool);
         425  +
    harness.shutdown().await.expect("clean harness shutdown");
         426  +
}
         427  +
         428  +
#[tokio::test]
         429  +
async fn acquisition_duration_ends_when_hyper_accepts_the_request() {
         430  +
    let response_gate = ManualGate::new();
         431  +
    let harness = ConnectionTestHarness::builder()
         432  +
        .endpoint(
         433  +
            IP1,
         434  +
            SocketScript::new()
         435  +
                .read_http1_request()
         436  +
                .wait(response_gate.waiter())
         437  +
                .write_all(
         438  +
                    "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: keep-alive\r\n\r\nok",
         439  +
                )
         440  +
                .await_client_close(),
         441  +
        )
         442  +
        .build()
         443  +
        .await
         444  +
        .expect("harness should start");
         445  +
    let pool_time = ManualTimeSource::new(UNIX_EPOCH);
         446  +
    let attempt_time = ManualTimeSource::new(UNIX_EPOCH);
         447  +
    let pool = ConnectionPool::builder()
         448  +
        .time_source(pool_time)
         449  +
        .build_http()
         450  +
        .expect("valid pool");
         451  +
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
         452  +
    let connector = test_client::connector_with_time_source(&client, attempt_time.clone());
         453  +
    let capture = CaptureHttpAttemptTelemetry::new();
         454  +
    let mut request = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         455  +
    request.add_extension(capture.clone());
         456  +
         457  +
    let send = tokio::spawn({
         458  +
        let connector = connector.clone();
         459  +
        async move { test_client::send_and_collect(&connector, request).await }
         460  +
    });
         461  +
    response_gate
         462  +
        .wait_until_reached(test_client::WAIT)
         463  +
        .await
         464  +
        .expect("server should receive the accepted request");
         465  +
    attempt_time.advance(Duration::from_secs(10));
         466  +
    response_gate.release();
         467  +
    let (status, body) = send.await.expect("request task");
         468  +
    assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         469  +
         470  +
    let telemetry = capture.get();
         471  +
    assert_eq!(
         472  +
        telemetry.acquisition().expect("acquisition").duration(),
         473  +
        Some(Duration::ZERO)
         474  +
    );
         475  +
    assert_eq!(
         476  +
        telemetry.connector_call_duration(),
         477  +
        Some(Duration::from_secs(10))
         478  +
    );
         479  +
         480  +
    drop(connector);
         481  +
    drop(client);
         482  +
    drop(pool);
         483  +
    harness.shutdown().await.expect("clean harness shutdown");
         484  +
}
         485  +
  322    486   
#[tokio::test]
  323    487   
async fn bounded_waiter_proceeds_after_the_active_h1_returns() {
  324    488   
    let body_gate = ManualGate::new();
  325    489   
    let harness = ConnectionTestHarness::builder()
  326    490   
        .endpoint(
  327    491   
            IP1,
  328    492   
            Http1Script::responses([
  329    493   
                Http1Response::ok().body_plan(BodyPlan::split_at_gate(
  330    494   
                    "first-",
  331    495   
                    body_gate.waiter(),
  332    496   
                    "body",
  333    497   
                )),
  334    498   
                Http1Response::ok().body("second"),
  335    499   
            ]),
  336    500   
        )
  337    501   
        .build()
  338    502   
        .await
  339    503   
        .expect("harness should start");
         504  +
    let attempt_time = ManualTimeSource::new(UNIX_EPOCH);
  340    505   
    let pool = ConnectionPool::builder()
  341    506   
        .max_connections_per_host(1)
  342    507   
        .build_http()
  343    508   
        .expect("valid pool");
  344    509   
    let client = SharedHttpClient::new(Client::new(&pool).expect("anonymous partition"));
  345         -
    let connector = connector(&client);
         510  +
    let connector = test_client::connector_with_time_source(&client, attempt_time.clone());
  346    511   
  347    512   
    let first = test_client::send_request(
  348    513   
        &connector,
  349    514   
        HttpRequest::get(harness.endpoint_url()).expect("valid request"),
  350    515   
    )
  351    516   
    .await
  352    517   
    .expect("first request should reach response headers");
  353    518   
    body_gate
  354    519   
        .wait_until_reached(test_client::WAIT)
  355    520   
        .await
  356    521   
        .expect("first response should reach its body gate");
  357    522   
         523  +
    let second_capture = CaptureHttpAttemptTelemetry::new();
         524  +
    let mut second_request = HttpRequest::get(harness.endpoint_url()).expect("valid request");
         525  +
    second_request.add_extension(second_capture.clone());
  358    526   
    let second_connector = connector.clone();
  359         -
    let second_url = harness.endpoint_url();
  360         -
    let mut second =
  361         -
        tokio::spawn(
  362         -
            async move { test_client::get_and_collect(&second_connector, &second_url).await },
  363         -
        );
         527  +
    let mut second = tokio::spawn(async move {
         528  +
        test_client::send_and_collect(&second_connector, second_request).await
         529  +
    });
  364    530   
    assert!(
  365    531   
        tokio::time::timeout(Duration::from_millis(100), &mut second)
  366    532   
            .await
  367    533   
            .is_err(),
  368    534   
        "the second request must wait while the only admitted H1 is active"
  369    535   
    );
  370    536   
    assert_eq!(1, harness.tcp_accepted_count());
  371    537   
         538  +
    attempt_time.advance(Duration::from_secs(5));
  372    539   
    body_gate.release();
  373    540   
    let (status, body) = test_client::collect_response(first).await;
  374    541   
    assert_eq!((status, body.as_slice()), (200, b"first-body".as_slice()));
  375    542   
    let (status, body) = second.await.expect("second request task should not panic");
  376    543   
    assert_eq!((status, body.as_slice()), (200, b"second".as_slice()));
  377    544   
    assert_eq!(1, harness.tcp_accepted_count());
         545  +
    let telemetry = second_capture.get();
         546  +
    let acquisition = telemetry.acquisition().expect("second acquisition");
         547  +
    assert_eq!(acquisition.duration(), Some(Duration::from_secs(5)));
         548  +
    assert_eq!(acquisition.usage(), ConnectionUsage::Reused);
         549  +
    assert_eq!(
         550  +
        telemetry.connector_call_duration(),
         551  +
        Some(Duration::from_secs(5))
         552  +
    );
  378    553   
  379    554   
    drop(connector);
  380    555   
    drop(client);
  381    556   
    drop(pool);
  382    557   
    harness.shutdown().await.expect("clean harness shutdown");
  383    558   
}
  384    559   
  385    560   
#[tokio::test]
  386    561   
async fn eligible_partition_borrows_the_peer_h1() {
  387    562   
    let harness = ConnectionTestHarness::builder()
@@ -410,585 +469,734 @@
  430    605   
    assert_eq!(1, harness.tcp_accepted_count());
  431    606   
  432    607   
    drop(first_connector);
  433    608   
    drop(second_connector);
  434    609   
    drop(first_client);
  435    610   
    drop(second_client);
  436    611   
    drop(pool);
  437    612   
    harness.shutdown().await.expect("clean harness shutdown");
  438    613   
}
  439    614   
         615  +
#[tokio::test]
         616  +
async fn peer_h1_reuse_survives_borrower_runtime_shutdown() {
         617  +
    use runtime::DrivenRuntime;
         618  +
         619  +
    let harness = ConnectionTestHarness::builder()
         620  +
        .endpoint(
         621  +
            IP1,
         622  +
            Http1Script::responses([
         623  +
                Http1Response::ok().body("owner"),
         624  +
                Http1Response::ok().body("peer"),
         625  +
                Http1Response::ok().body("reused"),
         626  +
            ]),
         627  +
        )
         628  +
        .build()
         629  +
        .await
         630  +
        .expect("harness should start");
         631  +
    let first_id = PartitionId::from_index(1);
         632  +
    let second_id = PartitionId::from_index(2);
         633  +
    let first_runtime = DrivenRuntime::start("h1-owner-one");
         634  +
    let second_runtime = DrivenRuntime::start("h1-owner-two");
         635  +
    assert_ne!(first_runtime.id(), second_runtime.id());
         636  +
    let pool = ConnectionPool::builder()
         637  +
        .partitions([
         638  +
            Partition::new(first_id, first_runtime.driver_spawner()),
         639  +
            Partition::new(second_id, second_runtime.driver_spawner()),
         640  +
        ])
         641  +
        .connection_reuse_scope(ConnectionReuseScope::Pool)
         642  +
        .max_connections_per_host(1)
         643  +
        .build_http()
         644  +
        .expect("valid pool");
         645  +
    let first_client = shared_client(&pool, first_id);
         646  +
    let second_client = shared_client(&pool, second_id);
         647  +
    let first_connector = connector(&first_client);
         648  +
    let second_connector = connector(&second_client);
         649  +
    let url = harness.endpoint_url();
         650  +
         651  +
    let first_request = first_connector.clone();
         652  +
    let first_url = url.clone();
         653  +
    let (status, body) = first_runtime
         654  +
        .spawn(async move { test_client::get_and_collect(&first_request, &first_url).await })
         655  +
        .await
         656  +
        .expect("owner request task should not panic");
         657  +
    assert_eq!((status, body.as_slice()), (200, b"owner".as_slice()));
         658  +
         659  +
    let peer_request = second_connector.clone();
         660  +
    let peer_url = url.clone();
         661  +
    let (status, body) = second_runtime
         662  +
        .spawn(async move { test_client::get_and_collect(&peer_request, &peer_url).await })
         663  +
        .await
         664  +
        .expect("peer request task should not panic");
         665  +
    assert_eq!((status, body.as_slice()), (200, b"peer".as_slice()));
         666  +
    assert!(first_runtime.submitted_tasks() > 0);
         667  +
    assert!(second_runtime.submitted_tasks() > 0);
         668  +
         669  +
    second_runtime.shutdown();
         670  +
         671  +
    let reused_request = first_connector.clone();
         672  +
    let reused_url = url.clone();
         673  +
    let (status, body) = first_runtime
         674  +
        .spawn(async move { test_client::get_and_collect(&reused_request, &reused_url).await })
         675  +
        .await
         676  +
        .expect("reused request task should not panic");
         677  +
    assert_eq!((status, body.as_slice()), (200, b"reused".as_slice()));
         678  +
         679  +
    let request_connections = harness
         680  +
        .events()
         681  +
        .into_iter()
         682  +
        .filter_map(|event| match event {
         683  +
            WireConnectionEvent::Http1Request { connection_id, .. } => Some(connection_id),
         684  +
            _ => None,
         685  +
        })
         686  +
        .collect::<Vec<_>>();
         687  +
    assert_eq!(3, request_connections.len());
         688  +
    assert!(
         689  +
        request_connections
         690  +
            .iter()
         691  +
            .all(|connection| *connection == request_connections[0]),
         692  +
        "owner and peer requests should use one HTTP/1 connection"
         693  +
    );
         694  +
    assert_eq!(1, harness.tcp_accepted_count());
         695  +
         696  +
    drop(first_connector);
         697  +
    drop(second_connector);
         698  +
    drop(first_client);
         699  +
    drop(second_client);
         700  +
    drop(pool);
         701  +
    first_runtime.shutdown();
         702  +
    harness.shutdown().await.expect("clean harness shutdown");
         703  +
}
         704  +
  440    705   
#[tokio::test]
  441    706   
async fn ineligible_partition_reclaims_peer_capacity() {
  442    707   
    let harness = ConnectionTestHarness::builder()
  443    708   
        .endpoint(
  444    709   
            IP1,
  445    710   
            EndpointPlan::queue([
  446    711   
                Http1Script::responses([Http1Response::ok().body("first")]),
  447    712   
                Http1Script::responses([Http1Response::ok().body("second")]),
  448    713   
            ]),
  449    714   
        )

tmp-codegen-diff/aws-sdk/sdk/aws-smithy-http-client/tests/h2_connection_behavior_test.rs

@@ -1,1 +53,58 @@
    4      4   
 */
    5      5   
    6      6   
//! HTTP/2 connection behavior contracts.
    7      7   
//!
    8      8   
//! Each contract is implementation-neutral and has an explicit runner for every
    9      9   
//! HTTP client backend expected to preserve that behavior.
   10     10   
   11     11   
#![cfg(all(feature = "wire-mock", feature = "rustls-aws-lc"))]
   12     12   
   13     13   
mod common;
          14  +
#[path = "common/runtime.rs"]
          15  +
mod runtime;
   14     16   
   15     17   
use aws_smithy_http_client::pool::{
   16     18   
    Client as PoolClient, ConnectionPool, ConnectionReuseScope, Partition, PartitionId,
   17     19   
    TokioDriverSpawner,
   18     20   
};
   19     21   
use aws_smithy_http_client::test_util::wire::connection::{ConnectionCloseReason, ManualGate};
   20     22   
use aws_smithy_http_client::tls;
   21     23   
use aws_smithy_runtime_api::client::connection::{
   22     24   
    CaptureSmithyConnection, ConnectionMetadata as SmithyConnectionMetadata,
   23     25   
};
          26  +
use aws_smithy_runtime_api::client::http::telemetry::{
          27  +
    CaptureHttpAttemptTelemetry, ConnectionUsage,
          28  +
};
   24     29   
use aws_smithy_runtime_api::client::http::{SharedHttpClient, SharedHttpConnector};
   25     30   
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
   26     31   
use bytes::Bytes;
   27     32   
use common::client as test_client;
   28     33   
use common::client::{
   29     34   
    BackendConfig, HttpsClientBackend, HyperUtilLegacyPool, PartitionedConnectionPool,
   30     35   
};
   31     36   
use common::h2::{
   32     37   
    H2BodyPlan, H2ConnectionId, H2ConnectionPlan, H2ConnectionScript, H2Event, H2Response,
   33     38   
    H2StreamScript, H2TestServer,
@@ -248,253 +307,443 @@
  268    273   
    }
  269    274   
  270    275   
    #[tokio::test]
  271    276   
    async fn test_concurrent_cold_start_converges_on_one_h2_connection_with_partitioned_pool() {
  272    277   
        concurrent_cold_start_converges_on_one_h2_connection(&PartitionedConnectionPool).await;
  273    278   
    }
  274    279   
}
  275    280   
  276    281   
mod connection_metadata {
  277    282   
    use super::*;
         283  +
    use aws_smithy_async::test_util::ManualTimeSource;
         284  +
    use std::time::{Duration, UNIX_EPOCH};
         285  +
         286  +
    async fn capture_attempt(
         287  +
        connector: &SharedHttpConnector,
         288  +
        url: &str,
         289  +
    ) -> aws_smithy_runtime_api::client::http::telemetry::HttpAttemptTelemetry {
         290  +
        let capture = CaptureHttpAttemptTelemetry::new();
         291  +
        let mut request = HttpRequest::get(url).expect("valid HTTP request");
         292  +
        request.add_extension(capture.clone());
         293  +
        let (status, body) = test_client::send_and_collect(connector, request).await;
         294  +
        assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         295  +
        capture.get()
         296  +
    }
         297  +
         298  +
    async fn attempt_telemetry_matches_backend_capability(
         299  +
        backend: &dyn HttpsClientBackend,
         300  +
        captures_selection: bool,
         301  +
    ) {
         302  +
        let server = H2TestServer::builder()
         303  +
            .connections(H2ConnectionPlan::queue([
         304  +
                H2ConnectionScript::new().fallback(H2StreamScript::respond(H2Response::ok("ok")))
         305  +
            ]))
         306  +
            .start()
         307  +
            .await
         308  +
            .expect("H2 server should start");
         309  +
        let client = h2_client(backend);
         310  +
        let connector = test_client::connector(&client);
         311  +
         312  +
        let first = capture_attempt(&connector, &server.url("/first")).await;
         313  +
        let second = capture_attempt(&connector, &server.url("/second")).await;
         314  +
        assert!(first.connector_call_duration().is_some());
         315  +
        assert!(second.connector_call_duration().is_some());
         316  +
         317  +
        if captures_selection {
         318  +
            assert_eq!(
         319  +
                first.acquisition().expect("first acquisition").usage(),
         320  +
                ConnectionUsage::Fresh
         321  +
            );
         322  +
            assert_eq!(
         323  +
                second.acquisition().expect("second acquisition").usage(),
         324  +
                ConnectionUsage::Reused
         325  +
            );
         326  +
            let first_connection = first.connection().expect("first connection metadata");
         327  +
            let second_connection = second.connection().expect("second connection metadata");
         328  +
            assert_eq!(
         329  +
                first_connection.connection_id(),
         330  +
                second_connection.connection_id()
         331  +
            );
         332  +
            assert_eq!(
         333  +
                first_connection.establishment(),
         334  +
                second_connection.establishment(),
         335  +
                "reuse must retain the establishment that created the connection"
         336  +
            );
         337  +
            assert!(first_connection.establishment().is_some());
         338  +
        } else {
         339  +
            assert!(first.acquisition().is_none());
         340  +
            assert!(first.connection().is_none());
         341  +
            assert!(second.acquisition().is_none());
         342  +
            assert!(second.connection().is_none());
         343  +
        }
         344  +
         345  +
        drop(connector);
         346  +
        drop(client);
         347  +
        server.shutdown().await.expect("clean H2 server shutdown");
         348  +
    }
         349  +
         350  +
    #[tokio::test]
         351  +
    async fn test_attempt_telemetry_with_hyper_util_legacy_pool() {
         352  +
        attempt_telemetry_matches_backend_capability(&HyperUtilLegacyPool, false).await;
         353  +
    }
         354  +
         355  +
    #[tokio::test]
         356  +
    async fn test_attempt_telemetry_with_partitioned_pool() {
         357  +
        attempt_telemetry_matches_backend_capability(&PartitionedConnectionPool, true).await;
         358  +
    }
         359  +
         360  +
    #[tokio::test]
         361  +
    async fn h2_acquisition_ends_before_response_headers() {
         362  +
        let response_gate = ManualGate::new();
         363  +
        let server = H2TestServer::builder()
         364  +
            .connections(H2ConnectionPlan::queue([H2ConnectionScript::new()
         365  +
                .fallback(H2StreamScript::respond_after(
         366  +
                    H2Response::ok("ok"),
         367  +
                    response_gate.waiter(),
         368  +
                ))]))
         369  +
            .start()
         370  +
            .await
         371  +
            .expect("H2 server should start");
         372  +
        let time = ManualTimeSource::new(UNIX_EPOCH);
         373  +
        let pool = ConnectionPool::builder()
         374  +
            .tls_provider(rustls_aws_lc())
         375  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
         376  +
            .time_source(time.clone())
         377  +
            .build_https()
         378  +
            .expect("valid partitioned HTTPS pool");
         379  +
        let client =
         380  +
            SharedHttpClient::new(PoolClient::new(&pool).expect("anonymous partition exists"));
         381  +
        let connector = test_client::connector_with_time_source(&client, time.clone());
         382  +
        let capture = CaptureHttpAttemptTelemetry::new();
         383  +
        let mut request = HttpRequest::get(server.url("/gated")).expect("valid HTTP request");
         384  +
        request.add_extension(capture.clone());
         385  +
        let send = tokio::spawn({
         386  +
            let connector = connector.clone();
         387  +
            async move { test_client::send_and_collect(&connector, request).await }
         388  +
        });
         389  +
         390  +
        response_gate
         391  +
            .wait_until_reached(test_client::WAIT)
         392  +
            .await
         393  +
            .expect("server should accept the H2 stream");
         394  +
        time.advance(Duration::from_secs(10));
         395  +
        response_gate.release();
         396  +
        let (status, body) = send.await.expect("request task");
         397  +
        assert_eq!((status, body.as_slice()), (200, b"ok".as_slice()));
         398  +
         399  +
        let telemetry = capture.get();
         400  +
        assert_eq!(
         401  +
            telemetry.acquisition().expect("acquisition").duration(),
         402  +
            Some(Duration::ZERO)
         403  +
        );
         404  +
        assert_eq!(
         405  +
            telemetry.connector_call_duration(),
         406  +
            Some(Duration::from_secs(10))
         407  +
        );
         408  +
         409  +
        drop(connector);
         410  +
        drop(client);
         411  +
        drop(pool);
         412  +
        server.shutdown().await.expect("clean H2 server shutdown");
         413  +
    }
  278    414   
  279    415   
    /// Poisoning captured H2 connection metadata moves later streams to a new connection.
  280    416   
    async fn poisoned_connection_is_not_reused(backend: &dyn HttpsClientBackend) {
  281    417   
        let script =
  282    418   
            H2ConnectionScript::new().fallback(H2StreamScript::respond(H2Response::ok("ok")));
  283    419   
        let server = H2TestServer::builder()
  284    420   
            .connections(H2ConnectionPlan::queue([script.clone(), script]))
  285    421   
            .start()
  286    422   
            .await
  287    423   
            .expect("H2 server should start");
@@ -612,748 +671,1050 @@
  632    768   
            "GOAWAY before dispatch created {} connections",
  633    769   
            server.connection_count()
  634    770   
        );
  635    771   
  636    772   
        drop(connector);
  637    773   
        drop(client);
  638    774   
        server.shutdown().await.expect("clean H2 server shutdown");
  639    775   
    }
  640    776   
}
  641    777   
         778  +
mod abrupt_transport_failure {
         779  +
    use super::*;
         780  +
    use aws_smithy_http_client::pool::ConnectionEvent;
         781  +
    use aws_smithy_runtime_api::client::connection::ConnectionId;
         782  +
    use std::sync::{Arc, Mutex};
         783  +
    use std::time::Duration;
         784  +
         785  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
         786  +
    enum PoolEvent {
         787  +
        Opened(ConnectionId),
         788  +
        LogicalClose(ConnectionId),
         789  +
        PhysicalClose(ConnectionId),
         790  +
    }
         791  +
         792  +
    /// Raw transport loss terminates every accepted stream and later demand uses
         793  +
    /// a replacement connection.
         794  +
    async fn abrupt_transport_failure_terminates_streams_and_replaces_connection(
         795  +
        client: SharedHttpClient,
         796  +
    ) -> aws_smithy_http_client::pool::OriginKey {
         797  +
        let body_gate = ManualGate::new();
         798  +
        let held = |body| {
         799  +
            H2StreamScript::respond(
         800  +
                H2Response::new(http_1x::StatusCode::OK).body(H2BodyPlan::gated(
         801  +
                    body,
         802  +
                    body_gate.waiter(),
         803  +
                    "-never",
         804  +
                )),
         805  +
            )
         806  +
        };
         807  +
        let first = H2ConnectionScript::new()
         808  +
            .route("/warm", H2StreamScript::respond(H2Response::ok("warm")))
         809  +
            .route("/one", held("one"))
         810  +
            .route("/two", held("two"))
         811  +
            .route("/three", held("three"));
         812  +
        let second = H2ConnectionScript::new().route(
         813  +
            "/replacement",
         814  +
            H2StreamScript::respond(H2Response::ok("replacement")),
         815  +
        );
         816  +
        let server = H2TestServer::builder()
         817  +
            .connections(H2ConnectionPlan::queue([first, second]))
         818  +
            .start()
         819  +
            .await
         820  +
            .expect("H2 server should start");
         821  +
        let connector = test_client::connector(&client);
         822  +
         823  +
        let (status, body) = test_client::get_and_collect(&connector, &server.url("/warm")).await;
         824  +
        assert_eq!((status, body.as_slice()), (200, b"warm".as_slice()));
         825  +
        let (one, two, three) = tokio::join!(
         826  +
            test_client::send_request(
         827  +
                &connector,
         828  +
                HttpRequest::get(server.url("/one")).expect("valid HTTP request"),
         829  +
            ),
         830  +
            test_client::send_request(
         831  +
                &connector,
         832  +
                HttpRequest::get(server.url("/two")).expect("valid HTTP request"),
         833  +
            ),
         834  +
            test_client::send_request(
         835  +
                &connector,
         836  +
                HttpRequest::get(server.url("/three")).expect("valid HTTP request"),
         837  +
            ),
         838  +
        );
         839  +
        let responses = [
         840  +
            one.expect("first response headers should arrive"),
         841  +
            two.expect("second response headers should arrive"),
         842  +
            three.expect("third response headers should arrive"),
         843  +
        ];
         844  +
        body_gate
         845  +
            .wait_for_arrivals(3, test_client::WAIT)
         846  +
            .await
         847  +
            .expect("all response streams should reach the body gate");
         848  +
         849  +
        let original = single_stream_connection(&server, "/warm");
         850  +
        assert_eq!(original, single_stream_connection(&server, "/one"));
         851  +
        assert_eq!(original, single_stream_connection(&server, "/two"));
         852  +
        assert_eq!(original, single_stream_connection(&server, "/three"));
         853  +
        server
         854  +
            .abort_transport(original)
         855  +
            .await
         856  +
            .expect("the original transport should abort");
         857  +
         858  +
        for response in responses {
         859  +
            let body_result =
         860  +
                tokio::time::timeout(test_client::WAIT, response.into_body().collect())
         861  +
                    .await
         862  +
                    .expect("aborted response body should terminate");
         863  +
            assert!(
         864  +
                body_result.is_err(),
         865  +
                "abrupt transport loss completed an accepted response body"
         866  +
            );
         867  +
        }
         868  +
        server
         869  +
            .wait_for_event(test_client::WAIT, |event| {
         870  +
                matches!(
         871  +
                    event,
         872  +
                    H2Event::ConnectionClosed {
         873  +
                        connection_id,
         874  +
                        reason: ConnectionCloseReason::ScriptedTransportAbort,
         875  +
                    } if *connection_id == original
         876  +
                )
         877  +
            })
         878  +
            .await
         879  +
            .expect("the harness should record raw transport loss");
         880  +
         881  +
        let replacement_url = server.url("/replacement");
         882  +
        let origin = aws_smithy_http_client::pool::OriginKey::from_uri(
         883  +
            &replacement_url.parse().expect("valid replacement URI"),
         884  +
        )
         885  +
        .expect("replacement URI should name an origin");
         886  +
        let (status, body) = test_client::get_and_collect(&connector, &replacement_url).await;
         887  +
        assert_eq!((status, body.as_slice()), (200, b"replacement".as_slice()));
         888  +
        assert_ne!(
         889  +
            original,
         890  +
            single_stream_connection(&server, "/replacement"),
         891  +
            "later demand reused the aborted connection"
         892  +
        );
         893  +
        assert_eq!(2, server.connection_count());
         894  +
         895  +
        drop(connector);
         896  +
        drop(client);
         897  +
        server.shutdown().await.expect("clean H2 server shutdown");
         898  +
        origin
         899  +
    }
         900  +
         901  +
    async fn wait_for_pool_events(
         902  +
        events: &Arc<Mutex<Vec<PoolEvent>>>,
         903  +
        predicate: impl Fn(&[PoolEvent]) -> bool,
         904  +
    ) {
         905  +
        tokio::time::timeout(test_client::WAIT, async {
         906  +
            loop {
         907  +
                if predicate(&events.lock().unwrap()) {
         908  +
                    return;
         909  +
                }
         910  +
                tokio::time::sleep(Duration::from_millis(10)).await;
         911  +
            }
         912  +
        })
         913  +
        .await
         914  +
        .expect("pool lifecycle events should converge");
         915  +
    }
         916  +
         917  +
    #[tokio::test]
         918  +
    async fn test_abrupt_transport_failure_terminates_streams_and_replaces_connection_with_hyper_util_legacy_pool(
         919  +
    ) {
         920  +
        abrupt_transport_failure_terminates_streams_and_replaces_connection(h2_client(
         921  +
            &HyperUtilLegacyPool,
         922  +
        ))
         923  +
        .await;
         924  +
    }
         925  +
         926  +
    #[tokio::test]
         927  +
    async fn test_abrupt_transport_failure_terminates_streams_and_replaces_connection_with_partitioned_pool(
         928  +
    ) {
         929  +
        let events = Arc::new(Mutex::new(Vec::new()));
         930  +
        let observed = events.clone();
         931  +
        let pool = ConnectionPool::builder()
         932  +
            .tls_provider(rustls_aws_lc())
         933  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
         934  +
            .max_connections_per_host(1)
         935  +
            .event_listener(move |event: &ConnectionEvent<'_>| {
         936  +
                let event = match event {
         937  +
                    ConnectionEvent::Opened(opened) => {
         938  +
                        Some(PoolEvent::Opened(opened.connection().id()))
         939  +
                    }
         940  +
                    ConnectionEvent::LogicalClose(closed) => {
         941  +
                        Some(PoolEvent::LogicalClose(closed.connection().id()))
         942  +
                    }
         943  +
                    ConnectionEvent::PhysicalClose(closed) => {
         944  +
                        Some(PoolEvent::PhysicalClose(closed.connection().id()))
         945  +
                    }
         946  +
                    ConnectionEvent::EstablishmentFailed(_) => None,
         947  +
                    _ => None,
         948  +
                };
         949  +
                if let Some(event) = event {
         950  +
                    observed.lock().unwrap().push(event);
         951  +
                }
         952  +
            })
         953  +
            .build_https()
         954  +
            .expect("valid partitioned HTTPS pool");
         955  +
        let client = SharedHttpClient::new(
         956  +
            PoolClient::new(&pool).expect("anonymous partition should resolve"),
         957  +
        );
         958  +
         959  +
        let origin =
         960  +
            abrupt_transport_failure_terminates_streams_and_replaces_connection(client).await;
         961  +
        wait_for_pool_events(&events, |events| {
         962  +
            events
         963  +
                .iter()
         964  +
                .filter(|event| matches!(event, PoolEvent::PhysicalClose(_)))
         965  +
                .count()
         966  +
                >= 1
         967  +
        })
         968  +
        .await;
         969  +
         970  +
        let events = events.lock().unwrap();
         971  +
        let opened: Vec<_> = events
         972  +
            .iter()
         973  +
            .filter_map(|event| match event {
         974  +
                PoolEvent::Opened(connection) => Some(*connection),
         975  +
                _ => None,
         976  +
            })
         977  +
            .collect();
         978  +
        assert_eq!(2, opened.len(), "expected original and replacement opens");
         979  +
        assert_ne!(opened[0], opened[1]);
         980  +
        assert_eq!(
         981  +
            1,
         982  +
            events
         983  +
                .iter()
         984  +
                .filter(|event| matches!(event, PoolEvent::LogicalClose(id) if *id == opened[0]))
         985  +
                .count(),
         986  +
            "the aborted generation should close logically once"
         987  +
        );
         988  +
        assert_eq!(
         989  +
            1,
         990  +
            events
         991  +
                .iter()
         992  +
                .filter(|event| matches!(event, PoolEvent::PhysicalClose(id) if *id == opened[0]))
         993  +
                .count(),
         994  +
            "the aborted transport should close physically once"
         995  +
        );
         996  +
        drop(events);
         997  +
         998  +
        let stats = pool
         999  +
            .partition_stats(PartitionId::ANONYMOUS, &origin)
        1000  +
            .expect("anonymous partition should retain origin statistics");
        1001  +
        assert_eq!(0, stats.pending_acquisitions());
        1002  +
        assert_eq!(0, stats.establishing_connections());
        1003  +
        assert_eq!(0, stats.h2().active_requests());
        1004  +
        assert_eq!(1, stats.h2().accepting());
        1005  +
        assert_eq!(0, stats.h2().draining());
        1006  +
        assert_eq!(1, stats.physically_live_connections());
        1007  +
        let capacity = pool
        1008  +
            .origin_stats(&origin)
        1009  +
            .capacity()
        1010  +
            .copied()
        1011  +
            .expect("origin should have bounded capacity");
        1012  +
        assert_eq!(1, capacity.limit());
        1013  +
        assert_eq!(
        1014  +
            1,
        1015  +
            capacity.in_use(),
        1016  +
            "replacement connection should own the returned slot"
        1017  +
        );
        1018  +
    }
        1019  +
}
        1020  +
  642   1021   
#[cfg(feature = "s2n-tls")]
  643   1022   
mod protocol_negotiation {
  644   1023   
    use super::*;
  645   1024   
  646   1025   
    /// The s2n-tls provider negotiates `h2` with ALPN and reuses that connection.
  647   1026   
    async fn s2n_negotiates_h2_and_reuses_connection(backend: &dyn HttpsClientBackend) {
  648   1027   
        let server = H2TestServer::builder()
  649   1028   
            .connections(H2ConnectionPlan::queue([H2ConnectionScript::new()
  650   1029   
                .fallback(H2StreamScript::respond(H2Response::ok("s2n-h2")))]))
  651   1030   
            .start()
@@ -1052,1431 +0,1697 @@
 1072   1451   
        let second_connection = single_stream_connection(&server, "/second");
 1073   1452   
        assert_ne!(first_connection, second_connection);
 1074   1453   
        assert_eq!(server.connection_count(), 2);
 1075   1454   
 1076   1455   
        drop(metadata);
 1077   1456   
        drop(first);
 1078   1457   
        drop(pool);
 1079   1458   
        server.shutdown().await.expect("clean H2 server shutdown");
 1080   1459   
    }
 1081   1460   
}
        1461  +
        1462  +
mod runtime_placement {
        1463  +
    use super::*;
        1464  +
    use aws_smithy_http_client::pool::ConnectionEvent;
        1465  +
    use aws_smithy_runtime_api::client::connection::ConnectionId;
        1466  +
    use runtime::DrivenRuntime;
        1467  +
    use std::sync::{Arc, Mutex};
        1468  +
    use std::time::Duration;
        1469  +
    use tokio::runtime::{Handle, Id};
        1470  +
        1471  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
        1472  +
    enum EventKind {
        1473  +
        Opened,
        1474  +
        LogicalClose,
        1475  +
        PhysicalClose,
        1476  +
    }
        1477  +
        1478  +
    #[derive(Clone, Copy, Debug, Eq, PartialEq)]
        1479  +
    struct ObservedEvent {
        1480  +
        kind: EventKind,
        1481  +
        connection: ConnectionId,
        1482  +
        partition: PartitionId,
        1483  +
        runtime: Option<Id>,
        1484  +
    }
        1485  +
        1486  +
    async fn wait_for_event(
        1487  +
        events: &Arc<Mutex<Vec<ObservedEvent>>>,
        1488  +
        predicate: impl Fn(&ObservedEvent) -> bool,
        1489  +
    ) -> ObservedEvent {
        1490  +
        tokio::time::timeout(test_client::WAIT, async {
        1491  +
            loop {
        1492  +
                if let Some(event) = events.lock().unwrap().iter().copied().find(&predicate) {
        1493  +
                    return event;
        1494  +
                }
        1495  +
                tokio::time::sleep(Duration::from_millis(10)).await;
        1496  +
            }
        1497  +
        })
        1498  +
        .await
        1499  +
        .expect("connection event should arrive")
        1500  +
    }
        1501  +
        1502  +
    #[tokio::test]
        1503  +
    async fn peer_h2_reuse_keeps_connection_work_on_the_owner_runtime() {
        1504  +
        let first_id = PartitionId::from_index(1);
        1505  +
        let second_id = PartitionId::from_index(2);
        1506  +
        let first_runtime = DrivenRuntime::start("h2-owner-one");
        1507  +
        let first_runtime_id = first_runtime.id();
        1508  +
        let second_runtime = DrivenRuntime::start("h2-owner-two");
        1509  +
        let second_runtime_id = second_runtime.id();
        1510  +
        let peer_body_gate = ManualGate::new();
        1511  +
        let first_script = H2ConnectionScript::new()
        1512  +
            .abort_streams_on_client_close()
        1513  +
            .route("/first", H2StreamScript::respond(H2Response::ok("first")))
        1514  +
            .route(
        1515  +
                "/peer",
        1516  +
                H2StreamScript::respond(
        1517  +
                    H2Response::new(http_1x::StatusCode::OK).body(H2BodyPlan::gated(
        1518  +
                        "peer-",
        1519  +
                        peer_body_gate.waiter(),
        1520  +
                        "never",
        1521  +
                    )),
        1522  +
                ),
        1523  +
            );
        1524  +
        let second_script = H2ConnectionScript::new().route(
        1525  +
            "/replacement",
        1526  +
            H2StreamScript::respond(H2Response::ok("replacement")),
        1527  +
        );
        1528  +
        let server = H2TestServer::builder()
        1529  +
            .connections(H2ConnectionPlan::queue([first_script, second_script]))
        1530  +
            .start()
        1531  +
            .await
        1532  +
            .expect("H2 server should start");
        1533  +
        1534  +
        let events = Arc::new(Mutex::new(Vec::new()));
        1535  +
        let observed = events.clone();
        1536  +
        let pool = ConnectionPool::builder()
        1537  +
            .tls_provider(rustls_aws_lc())
        1538  +
            .tls_context(test_tls::SERVER_IDENTITY.client_context())
        1539  +
            .partitions([
        1540  +
                Partition::new(first_id, first_runtime.driver_spawner()),
        1541  +
                Partition::new(second_id, second_runtime.driver_spawner()),
        1542  +
            ])
        1543  +
            .connection_reuse_scope(ConnectionReuseScope::Pool)
        1544  +
            .max_connections_per_host(1)
        1545  +
            .event_listener(move |event: &ConnectionEvent<'_>| {
        1546  +
                let observed_event = match event {
        1547  +
                    ConnectionEvent::Opened(opened) => {
        1548  +
                        Some((EventKind::Opened, opened.connection()))
        1549  +
                    }
        1550  +
                    ConnectionEvent::LogicalClose(closed) => {
        1551  +
                        Some((EventKind::LogicalClose, closed.connection()))
        1552  +
                    }
        1553  +
                    ConnectionEvent::PhysicalClose(closed) => {
        1554  +
                        Some((EventKind::PhysicalClose, closed.connection()))
        1555  +
                    }
        1556  +
                    ConnectionEvent::EstablishmentFailed(_) => None,
        1557  +
                    _ => None,
        1558  +
                };
        1559  +
                if let Some((kind, connection)) = observed_event {
        1560  +
                    observed.lock().unwrap().push(ObservedEvent {
        1561  +
                        kind,
        1562  +
                        connection: connection.id(),
        1563  +
                        partition: connection.owner_partition(),
        1564  +
                        runtime: Handle::try_current().ok().map(|handle| handle.id()),
        1565  +
                    });
        1566  +
                }
        1567  +
            })
        1568  +
            .build_https()
        1569  +
            .expect("valid partitioned HTTPS pool");
        1570  +
        let first_client = SharedHttpClient::new(
        1571  +
            PoolClient::from_partition(&pool, first_id).expect("first partition should resolve"),
        1572  +
        );
        1573  +
        let second_client = SharedHttpClient::new(
        1574  +
            PoolClient::from_partition(&pool, second_id).expect("second partition should resolve"),
        1575  +
        );
        1576  +
        let first = test_client::connector(&first_client);
        1577  +
        let second = test_client::connector(&second_client);
        1578  +
        1579  +
        let first_request = first.clone();
        1580  +
        let first_url = server.url("/first");
        1581  +
        let (status, body) = first_runtime
        1582  +
            .spawn(async move { test_client::get_and_collect(&first_request, &first_url).await })
        1583  +
            .await
        1584  +
            .expect("first partition request task should not panic");
        1585  +
        assert_eq!((status, body.as_slice()), (200, b"first".as_slice()));
        1586  +
        let original_server_connection = single_stream_connection(&server, "/first");
        1587  +
        let original_open = wait_for_event(&events, |event| {
        1588  +
            event.kind == EventKind::Opened && event.partition == first_id
        1589  +
        })
        1590  +
        .await;
        1591  +
        assert_eq!(Some(first_runtime_id), original_open.runtime);
        1592  +
        assert!(first_runtime.submitted_tasks() > 0);
        1593  +
        1594  +
        let peer_request = second.clone();
        1595  +
        let peer_url = server.url("/peer");
        1596  +
        let peer_response = second_runtime
        1597  +
            .spawn(async move {
        1598  +
                test_client::send_request(
        1599  +
                    &peer_request,
        1600  +
                    HttpRequest::get(peer_url).expect("valid HTTP request"),
        1601  +
                )
        1602  +
                .await
        1603  +
            })
        1604  +
            .await
        1605  +
            .expect("peer request task should not panic")
        1606  +
            .expect("peer response headers should arrive");
        1607  +
        peer_body_gate
        1608  +
            .wait_until_reached(test_client::WAIT)
        1609  +
            .await
        1610  +
            .expect("peer response should reach its body gate");
        1611  +
        assert_eq!(
        1612  +
            original_server_connection,
        1613  +
            single_stream_connection(&server, "/peer"),
        1614  +
            "peer demand should use the first partition's generation"
        1615  +
        );
        1616  +
        assert!(
        1617  +
            !events
        1618  +
                .lock()
        1619  +
                .unwrap()
        1620  +
                .iter()
        1621  +
                .any(|event| { event.kind == EventKind::Opened && event.partition == second_id }),
        1622  +
            "peer reuse should not establish a requester-owned connection"
        1623  +
        );
        1624  +
        1625  +
        first_runtime.shutdown();
        1626  +
        let body_result =
        1627  +
            tokio::time::timeout(test_client::WAIT, peer_response.into_body().collect())
        1628  +
                .await
        1629  +
                .expect("owner-runtime shutdown should terminate the accepted response");
        1630  +
        assert!(
        1631  +
            body_result.is_err(),
        1632  +
            "accepted response completed after its owner runtime stopped"
        1633  +
        );
        1634  +
        let logical_close = wait_for_event(&events, |event| {
        1635  +
            event.kind == EventKind::LogicalClose && event.connection == original_open.connection
        1636  +
        })
        1637  +
        .await;
        1638  +
        assert_eq!(first_id, logical_close.partition);
        1639  +
        let physical_close = wait_for_event(&events, |event| {
        1640  +
            event.kind == EventKind::PhysicalClose && event.connection == original_open.connection
        1641  +
        })
        1642  +
        .await;
        1643  +
        assert_eq!(first_id, physical_close.partition);
        1644  +
        1645  +
        let replacement_request = second.clone();
        1646  +
        let replacement_url = server.url("/replacement");
        1647  +
        let (status, body) = second_runtime
        1648  +
            .spawn(async move {
        1649  +
                test_client::get_and_collect(&replacement_request, &replacement_url).await
        1650  +
            })
        1651  +
            .await
        1652  +
            .expect("replacement request task should not panic");
        1653  +
        assert_eq!((status, body.as_slice()), (200, b"replacement".as_slice()));
        1654  +
        assert_ne!(
        1655  +
            original_server_connection,
        1656  +
            single_stream_connection(&server, "/replacement")
        1657  +
        );
        1658  +
        let replacement_open = wait_for_event(&events, |event| {
        1659  +
            event.kind == EventKind::Opened && event.partition == second_id
        1660  +
        })
        1661  +
        .await;
        1662  +
        assert_eq!(Some(second_runtime_id), replacement_open.runtime);
        1663  +
        assert_ne!(original_open.connection, replacement_open.connection);
        1664  +
        assert!(second_runtime.submitted_tasks() > 0);
        1665  +
        1666  +
        let origin = aws_smithy_http_client::pool::OriginKey::from_uri(
        1667  +
            &server.url("/").parse().expect("valid server URI"),
        1668  +
        )
        1669  +
        .expect("server URI should name an origin");
        1670  +
        let first_stats = pool
        1671  +
            .partition_stats(first_id, &origin)
        1672  +
            .expect("first partition should retain origin statistics");
        1673  +
        assert_eq!(0, first_stats.h2().active_requests());
        1674  +
        assert_eq!(0, first_stats.physically_live_connections());
        1675  +
        let second_stats = pool
        1676  +
            .partition_stats(second_id, &origin)
        1677  +
            .expect("second partition should retain origin statistics");
        1678  +
        assert_eq!(1, second_stats.h2().accepting());
        1679  +
        assert_eq!(0, second_stats.h2().active_requests());
        1680  +
        assert_eq!(1, second_stats.physically_live_connections());
        1681  +
        let capacity = pool
        1682  +
            .origin_stats(&origin)
        1683  +
            .capacity()
        1684  +
            .copied()
        1685  +
            .expect("origin should have bounded capacity");
        1686  +
        assert_eq!(1, capacity.limit());
        1687  +
        assert_eq!(1, capacity.in_use());
        1688  +
        1689  +
        drop(first);
        1690  +
        drop(second);
        1691  +
        drop(first_client);
        1692  +
        drop(second_client);
        1693  +
        drop(pool);
        1694  +
        second_runtime.shutdown();
        1695  +
        server.shutdown().await.expect("clean H2 server shutdown");
        1696  +
    }
        1697  +
}