aws_smithy_json/deserialize/
token.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use crate::deserialize::error::DeserializeError as Error;
7use crate::deserialize::must_not_be_finite;
8use crate::escape::unescape_string;
9pub use crate::escape::EscapeError;
10use aws_smithy_types::date_time::Format;
11use aws_smithy_types::primitive::Parse;
12use aws_smithy_types::{base64, Blob, DateTime, Document, Number};
13use std::borrow::Cow;
14use std::collections::HashMap;
15use std::iter::Peekable;
16
17/// New-type around `&str` that indicates the string is an escaped JSON string.
18/// Provides functions for retrieving the string in either form.
19#[derive(Debug, PartialEq, Eq, Copy, Clone)]
20pub struct EscapedStr<'a>(&'a str);
21
22impl<'a> EscapedStr<'a> {
23    pub fn new(value: &'a str) -> EscapedStr<'a> {
24        EscapedStr(value)
25    }
26
27    /// Returns the escaped string value
28    pub fn as_escaped_str(&self) -> &'a str {
29        self.0
30    }
31
32    /// Unescapes the string and returns it.
33    /// If the string doesn't need unescaping, it will be returned directly.
34    pub fn to_unescaped(self) -> Result<Cow<'a, str>, EscapeError> {
35        unescape_string(self.0)
36    }
37}
38
39/// Represents the location of a token
40#[derive(Debug, Eq, PartialEq, Copy, Clone)]
41pub struct Offset(pub usize);
42
43impl Offset {
44    /// Creates a custom error from the offset
45    pub fn error(&self, msg: Cow<'static, str>) -> Error {
46        Error::custom(msg).with_offset(self.0)
47    }
48}
49
50/// Enum representing the different JSON tokens that can be returned by
51/// [`crate::deserialize::json_token_iter`].
52#[derive(Debug, PartialEq)]
53pub enum Token<'a> {
54    StartArray {
55        offset: Offset,
56    },
57    EndArray {
58        offset: Offset,
59    },
60    ObjectKey {
61        offset: Offset,
62        key: EscapedStr<'a>,
63    },
64    StartObject {
65        offset: Offset,
66    },
67    EndObject {
68        offset: Offset,
69    },
70    ValueBool {
71        offset: Offset,
72        value: bool,
73    },
74    ValueNull {
75        offset: Offset,
76    },
77    ValueNumber {
78        offset: Offset,
79        value: Number,
80    },
81    ValueString {
82        offset: Offset,
83        value: EscapedStr<'a>,
84    },
85}
86
87impl Token<'_> {
88    pub fn offset(&self) -> Offset {
89        use Token::*;
90        *match self {
91            StartArray { offset } => offset,
92            EndArray { offset } => offset,
93            ObjectKey { offset, .. } => offset,
94            StartObject { offset } => offset,
95            EndObject { offset } => offset,
96            ValueBool { offset, .. } => offset,
97            ValueNull { offset } => offset,
98            ValueNumber { offset, .. } => offset,
99            ValueString { offset, .. } => offset,
100        }
101    }
102
103    /// Builds an error from the token's offset
104    pub fn error(&self, msg: Cow<'static, str>) -> Error {
105        self.offset().error(msg)
106    }
107}
108
109macro_rules! expect_fn {
110    ($name:ident, $token:ident, $doc:tt) => {
111        #[doc=$doc]
112        pub fn $name(token_result: Option<Result<Token<'_>, Error>>) -> Result<(), Error> {
113            match token_result.transpose()? {
114                Some(Token::$token { .. }) => Ok(()),
115                Some(token) => {
116                    Err(token.error(Cow::Borrowed(concat!("expected ", stringify!($token)))))
117                }
118                None => Err(Error::custom(concat!("expected ", stringify!($token)))),
119            }
120        }
121    };
122}
123
124expect_fn!(
125    expect_start_object,
126    StartObject,
127    "Expects a [Token::StartObject] token and returns an error if it's not present."
128);
129expect_fn!(
130    expect_start_array,
131    StartArray,
132    "Expects a [Token::StartArray] token and returns an error if it's not present."
133);
134
135macro_rules! expect_value_or_null_fn {
136    ($name:ident, $token:ident, $typ:ident, $doc:tt) => {
137        #[doc=$doc]
138        #[allow(unknown_lints)]
139        #[allow(mismatched_lifetime_syntaxes)]
140        pub fn $name(token: Option<Result<Token<'_>, Error>>) -> Result<Option<$typ>, Error> {
141            match token.transpose()? {
142                Some(Token::ValueNull { .. }) => Ok(None),
143                Some(Token::$token { value, .. }) => Ok(Some(value)),
144                _ => Err(Error::custom(concat!(
145                    "expected ",
146                    stringify!($token),
147                    " or ValueNull"
148                ))),
149            }
150        }
151    };
152}
153
154expect_value_or_null_fn!(expect_bool_or_null, ValueBool, bool, "Expects a [Token::ValueBool] or [Token::ValueNull], and returns the bool value if it's not null.");
155expect_value_or_null_fn!(expect_string_or_null, ValueString, EscapedStr, "Expects a [Token::ValueString] or [Token::ValueNull], and returns the [EscapedStr] value if it's not null.");
156
157/// Expects a [Token::ValueString], [Token::ValueNumber] or [Token::ValueNull].
158///
159/// If the value is a string, it MUST be `Infinity`, `-Infinity` or `Nan`.
160/// If the value is a number, it is returned directly
161pub fn expect_number_or_null(
162    token: Option<Result<Token<'_>, Error>>,
163) -> Result<Option<Number>, Error> {
164    match token.transpose()? {
165        Some(Token::ValueNull { .. }) => Ok(None),
166        Some(Token::ValueNumber { value, offset, .. }) => {
167            // Validate finite numbers - error on infinity/NaN
168            match value {
169                Number::Float(f) if !f.is_finite() => {
170                    Err(Error::custom("number must be finite").with_offset(offset.0))
171                }
172                _ => Ok(Some(value)),
173            }
174        }
175        Some(Token::ValueString { value, offset }) => match value.to_unescaped() {
176            Err(err) => Err(Error::custom_source( "expected a valid string, escape was invalid", err).with_offset(offset.0)),
177            Ok(v) => f64::parse_smithy_primitive(v.as_ref())
178                // disregard the exact error
179                .map_err(|_|())
180                // only infinite / NaN can be used as strings
181                .and_then(must_not_be_finite)
182                .map(|float| Some(aws_smithy_types::Number::Float(float)))
183                // convert to a helpful error
184                .map_err(|_| {
185                    Error::custom(
186                        format!(
187                        "only `Infinity`, `-Infinity`, `NaN` can represent a float as a string but found `{v}`"
188                    )).with_offset(offset.0)
189                }),
190        },
191        _ => Err(Error::custom(
192            "expected ValueString, ValueNumber, or ValueNull",
193        )),
194    }
195}
196
197/// Expects a [Token::ValueNumber] or [Token::ValueNull], and returns the number as a string
198/// to preserve arbitrary precision.
199///
200/// This function extracts the raw JSON number string without converting it to u64/i64/f64,
201/// which would cause precision loss for numbers larger than those types can represent.
202/// This is essential for BigInteger and BigDecimal support.
203///
204/// # Arguments
205/// * `token` - The token to extract the number from
206/// * `input` - The original JSON input bytes (needed to extract the raw number string)
207///
208/// # Returns
209/// * `Ok(Some(string))` - The number as a string slice
210/// * `Ok(None)` - If the token is null
211/// * `Err` - If the token is not a number or null
212pub fn expect_number_as_string_or_null<'a>(
213    token: Option<Result<Token<'a>, Error>>,
214    input: &'a [u8],
215) -> Result<Option<&'a str>, Error> {
216    match token.transpose()? {
217        Some(Token::ValueNull { .. }) => Ok(None),
218        Some(Token::ValueNumber { offset, .. }) => {
219            let start = offset.0;
220            let mut end = start;
221
222            // Skip optional minus sign
223            if end < input.len() && input[end] == b'-' {
224                end += 1;
225            }
226
227            // Scan digits, decimal point, exponent
228            while end < input.len() {
229                match input[end] {
230                    b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-' => end += 1,
231                    _ => break,
232                }
233            }
234
235            let number_slice = &input[start..end];
236            let number_str = std::str::from_utf8(number_slice)
237                .map_err(|_| Error::custom("invalid UTF-8 in number"))?;
238            Ok(Some(number_str))
239        }
240        _ => Err(Error::custom("expected ValueNumber or ValueNull")),
241    }
242}
243
244/// Expects a [Token::ValueString] or [Token::ValueNull]. If the value is a string, it interprets it as a base64 encoded [Blob] value.
245pub fn expect_blob_or_null(token: Option<Result<Token<'_>, Error>>) -> Result<Option<Blob>, Error> {
246    Ok(match expect_string_or_null(token)? {
247        Some(value) => Some(Blob::new(
248            base64::decode(value.as_escaped_str())
249                .map_err(|err| Error::custom_source("failed to decode base64", err))?,
250        )),
251        None => None,
252    })
253}
254
255/// Expects a [Token::ValueNull], [Token::ValueString], or [Token::ValueNumber] depending
256/// on the passed in `timestamp_format`. If there is a non-null value, it interprets it as an
257/// [`DateTime` ] in the requested format.
258pub fn expect_timestamp_or_null(
259    token: Option<Result<Token<'_>, Error>>,
260    timestamp_format: Format,
261) -> Result<Option<DateTime>, Error> {
262    Ok(match timestamp_format {
263        Format::EpochSeconds => expect_number_or_null(token)?
264            .map(|v| v.to_f64_lossy())
265            .map(|v| {
266                if v.is_nan() {
267                    Err(Error::custom("NaN is not a valid epoch"))
268                } else if v.is_infinite() {
269                    Err(Error::custom("infinity is not a valid epoch"))
270                } else {
271                    Ok(DateTime::from_secs_f64(v))
272                }
273            })
274            .transpose()?,
275        Format::DateTime | Format::HttpDate | Format::DateTimeWithOffset => {
276            expect_string_or_null(token)?
277                .map(|v| DateTime::from_str(v.as_escaped_str(), timestamp_format))
278                .transpose()
279                .map_err(|err| Error::custom_source("failed to parse timestamp", err))?
280        }
281    })
282}
283
284/// Expects and parses a complete document value.
285pub fn expect_document<'a, I>(tokens: &mut Peekable<I>) -> Result<Document, Error>
286where
287    I: Iterator<Item = Result<Token<'a>, Error>>,
288{
289    expect_document_inner(tokens, 0)
290}
291
292const MAX_DOCUMENT_RECURSION: usize = 256;
293
294fn expect_document_inner<'a, I>(tokens: &mut Peekable<I>, depth: usize) -> Result<Document, Error>
295where
296    I: Iterator<Item = Result<Token<'a>, Error>>,
297{
298    if depth >= MAX_DOCUMENT_RECURSION {
299        return Err(Error::custom(
300            "exceeded max recursion depth while parsing document",
301        ));
302    }
303    match tokens.next().transpose()? {
304        Some(Token::ValueNull { .. }) => Ok(Document::Null),
305        Some(Token::ValueBool { value, .. }) => Ok(Document::Bool(value)),
306        Some(Token::ValueNumber { value, .. }) => Ok(Document::Number(value)),
307        Some(Token::ValueString { value, .. }) => {
308            Ok(Document::String(value.to_unescaped()?.into_owned()))
309        }
310        Some(Token::StartObject { .. }) => {
311            let mut object = HashMap::new();
312            loop {
313                match tokens.next().transpose()? {
314                    Some(Token::EndObject { .. }) => break,
315                    Some(Token::ObjectKey { key, .. }) => {
316                        let key = key.to_unescaped()?.into_owned();
317                        let value = expect_document_inner(tokens, depth + 1)?;
318                        object.insert(key, value);
319                    }
320                    _ => return Err(Error::custom("expected object key or end object")),
321                }
322            }
323            Ok(Document::Object(object))
324        }
325        Some(Token::StartArray { .. }) => {
326            let mut array = Vec::new();
327            loop {
328                match tokens.peek() {
329                    Some(Ok(Token::EndArray { .. })) => {
330                        tokens.next().transpose().unwrap();
331                        break;
332                    }
333                    _ => array.push(expect_document_inner(tokens, depth + 1)?),
334                }
335            }
336            Ok(Document::Array(array))
337        }
338        Some(Token::EndObject { .. }) | Some(Token::ObjectKey { .. }) => {
339            unreachable!("end object and object key are handled in start object")
340        }
341        Some(Token::EndArray { .. }) => unreachable!("end array is handled in start array"),
342        None => Err(Error::custom("expected value")),
343    }
344}
345
346/// Skips an entire value in the token stream. Errors if it isn't a value.
347pub fn skip_value<'a>(
348    tokens: &mut impl Iterator<Item = Result<Token<'a>, Error>>,
349) -> Result<(), Error> {
350    skip_inner(0, tokens)
351}
352
353/// Assumes a start object/array token has already been consumed and skips tokens until
354/// until its corresponding end object/array token is found.
355pub fn skip_to_end<'a>(
356    tokens: &mut impl Iterator<Item = Result<Token<'a>, Error>>,
357) -> Result<(), Error> {
358    skip_inner(1, tokens)
359}
360
361/// Maximum nesting depth allowed while skipping a JSON value.
362const MAX_SKIP_DEPTH: usize = 512;
363
364fn skip_inner<'a>(
365    initial_depth: usize,
366    tokens: &mut impl Iterator<Item = Result<Token<'a>, Error>>,
367) -> Result<(), Error> {
368    let mut depth = initial_depth;
369    loop {
370        match tokens.next().transpose()? {
371            Some(Token::StartObject { .. }) | Some(Token::StartArray { .. }) => {
372                depth = depth.checked_add(1).ok_or_else(|| {
373                    Error::custom("exceeded max recursion depth while skipping value")
374                })?;
375                if depth > MAX_SKIP_DEPTH {
376                    return Err(Error::custom(
377                        "exceeded max recursion depth while skipping value",
378                    ));
379                }
380            }
381            Some(Token::EndObject { .. }) | Some(Token::EndArray { .. }) => {
382                debug_assert!(depth > 0);
383                // The token iterator validates matching braces, so reaching
384                // this branch with depth == 0 indicates a bug upstream rather
385                // than untrusted input. Saturate to avoid underflow in
386                // release builds.
387                depth = depth.saturating_sub(1);
388                if depth == 0 {
389                    break;
390                }
391            }
392            Some(Token::ValueNull { .. })
393            | Some(Token::ValueBool { .. })
394            | Some(Token::ValueNumber { .. })
395            | Some(Token::ValueString { .. }) => {
396                if depth == 0 {
397                    break;
398                }
399            }
400            Some(Token::ObjectKey { .. }) => {}
401            None => return Err(Error::custom("expected value")),
402        }
403    }
404    Ok(())
405}
406
407#[cfg(test)]
408pub mod test {
409    use super::*;
410    use crate::deserialize::error::DeserializeErrorKind as ErrorKind;
411    use crate::deserialize::error::DeserializeErrorKind::UnexpectedToken;
412    use crate::deserialize::json_token_iter;
413
414    pub fn start_array<'a>(offset: usize) -> Option<Result<Token<'a>, Error>> {
415        Some(Ok(Token::StartArray {
416            offset: Offset(offset),
417        }))
418    }
419
420    pub fn end_array<'a>(offset: usize) -> Option<Result<Token<'a>, Error>> {
421        Some(Ok(Token::EndArray {
422            offset: Offset(offset),
423        }))
424    }
425
426    pub fn start_object<'a>(offset: usize) -> Option<Result<Token<'a>, Error>> {
427        Some(Ok(Token::StartObject {
428            offset: Offset(offset),
429        }))
430    }
431
432    pub fn end_object<'a>(offset: usize) -> Option<Result<Token<'a>, Error>> {
433        Some(Ok(Token::EndObject {
434            offset: Offset(offset),
435        }))
436    }
437
438    pub fn object_key(offset: usize, key: &str) -> Option<Result<Token<'_>, Error>> {
439        Some(Ok(Token::ObjectKey {
440            offset: Offset(offset),
441            key: EscapedStr::new(key),
442        }))
443    }
444
445    pub fn value_bool<'a>(offset: usize, boolean: bool) -> Option<Result<Token<'a>, Error>> {
446        Some(Ok(Token::ValueBool {
447            offset: Offset(offset),
448            value: boolean,
449        }))
450    }
451
452    pub fn value_number<'a>(offset: usize, number: Number) -> Option<Result<Token<'a>, Error>> {
453        Some(Ok(Token::ValueNumber {
454            offset: Offset(offset),
455            value: number,
456        }))
457    }
458
459    pub fn value_null<'a>(offset: usize) -> Option<Result<Token<'a>, Error>> {
460        Some(Ok(Token::ValueNull {
461            offset: Offset(offset),
462        }))
463    }
464
465    pub fn value_string(offset: usize, string: &str) -> Option<Result<Token<'_>, Error>> {
466        Some(Ok(Token::ValueString {
467            offset: Offset(offset),
468            value: EscapedStr::new(string),
469        }))
470    }
471
472    #[track_caller]
473    fn expect_err_custom<T>(message: &str, offset: Option<usize>, result: Result<T, Error>) {
474        let err = result.err().expect("expected error");
475        let (actual_message, actual_offset) = match &err.kind {
476            ErrorKind::Custom { message, .. } => (message.as_ref(), err.offset),
477            _ => panic!("expected ErrorKind::Custom, got {err:?}"),
478        };
479        assert_eq!((message, offset), (actual_message, actual_offset));
480    }
481
482    #[test]
483    fn skip_simple_value() {
484        let mut tokens = json_token_iter(b"null true");
485        skip_value(&mut tokens).unwrap();
486        assert!(matches!(
487            tokens.next(),
488            Some(Ok(Token::ValueBool { value: true, .. }))
489        ))
490    }
491
492    #[test]
493    fn skip_array() {
494        let mut tokens = json_token_iter(b"[1, 2, 3, 4] true");
495        skip_value(&mut tokens).unwrap();
496        assert!(matches!(
497            tokens.next(),
498            Some(Ok(Token::ValueBool { value: true, .. }))
499        ))
500    }
501
502    #[test]
503    fn skip_object() {
504        let mut tokens = json_token_iter(b"{\"one\": 5, \"two\": 3} true");
505        skip_value(&mut tokens).unwrap();
506        assert!(matches!(
507            tokens.next(),
508            Some(Ok(Token::ValueBool { value: true, .. }))
509        ))
510    }
511
512    #[test]
513    fn test_skip_to_end() {
514        let tokens = json_token_iter(b"{\"one\": { \"two\": [] }, \"three\":2 }");
515        let mut tokens = tokens.skip(2);
516        assert!(matches!(tokens.next(), Some(Ok(Token::StartObject { .. }))));
517        skip_to_end(&mut tokens).unwrap();
518        match tokens.next() {
519            Some(Ok(Token::ObjectKey { key, .. })) => {
520                assert_eq!("three", key.as_escaped_str());
521            }
522            _ => panic!("expected object key three"),
523        }
524    }
525
526    #[test]
527    fn test_non_finite_floats() {
528        let mut tokens = json_token_iter(b"inf");
529        tokens
530            .next()
531            .expect("there is a token")
532            .expect_err("but it is invalid, ensure that Rust float boundary cases don't parse");
533    }
534
535    #[test]
536    fn mismatched_braces() {
537        // The skip_value function doesn't need to explicitly handle these cases since
538        // token iterator's parser handles them. This test confirms that assumption.
539        assert!(matches!(
540            skip_value(&mut json_token_iter(br#"[{"foo": 5]}"#)),
541            Err(Error {
542                kind: UnexpectedToken(']', "'}', ','"),
543                offset: Some(10)
544            })
545        ));
546        assert!(matches!(
547            skip_value(&mut json_token_iter(br#"{"foo": 5]}"#)),
548            Err(Error {
549                kind: UnexpectedToken(']', "'}', ','"),
550                offset: Some(9)
551            })
552        ));
553        assert!(matches!(
554            skip_value(&mut json_token_iter(br#"[5,6}"#)),
555            Err(Error {
556                kind: UnexpectedToken('}', "']', ','"),
557                offset: Some(4)
558            })
559        ));
560    }
561
562    #[test]
563    fn skip_nested() {
564        let mut tokens = json_token_iter(
565            br#"
566            {"struct": {"foo": 5, "bar": 11, "arr": [1, 2, 3, {}, 5, []]},
567             "arr": [[], [[]], [{"arr":[]}]],
568             "simple": "foo"}
569            true
570        "#,
571        );
572        skip_value(&mut tokens).unwrap();
573        assert!(matches!(
574            tokens.next(),
575            Some(Ok(Token::ValueBool { value: true, .. }))
576        ))
577    }
578
579    /// Regression test for unbounded recursion in `skip_inner`.
580    ///
581    /// The previous implementation recursed once per nested container. A payload
582    /// consisting of hundreds of thousands of `[` characters therefore recursed
583    /// deeply enough to overflow the default thread stack. With the iterative
584    /// implementation we instead return a bounded depth-limit error.
585    #[test]
586    fn skip_deeply_nested_returns_depth_error() {
587        // Far deeper than MAX_SKIP_DEPTH, and far deeper than a typical
588        // thread stack can tolerate with a recursive implementation
589        // (~8 bytes per recursion would blow 8 MiB at ~1M frames, and the
590        // real frame cost is much larger than that in debug builds).
591        let depth = 200_000;
592        let mut payload = Vec::with_capacity(depth * 2);
593        payload.extend(std::iter::repeat_n(b'[', depth));
594        payload.extend(std::iter::repeat_n(b']', depth));
595
596        let mut tokens = json_token_iter(&payload);
597        let err = skip_value(&mut tokens).expect_err("should hit the depth limit");
598        match err.kind {
599            ErrorKind::Custom { message, .. } => {
600                assert!(
601                    message.contains("exceeded max recursion depth"),
602                    "unexpected error message: {message}"
603                );
604            }
605            other => panic!("expected Custom depth-limit error, got {other:?}"),
606        }
607    }
608
609    /// The same scenario as [`skip_deeply_nested_returns_depth_error`], but run
610    /// on a thread with a very small stack. The previous recursive
611    /// implementation would stack-overflow here (aborting the process).
612    /// The iterative implementation uses O(1) stack so this completes with a
613    /// normal error return value.
614    #[test]
615    fn skip_deeply_nested_does_not_overflow_stack() {
616        // 256 KiB — too small for hundreds of thousands of recursive frames,
617        // plenty for the iterative implementation.
618        const SMALL_STACK: usize = 256 * 1024;
619
620        let handle = std::thread::Builder::new()
621            .stack_size(SMALL_STACK)
622            .spawn(|| {
623                let depth = 200_000;
624                let mut payload = Vec::with_capacity(depth * 2);
625                payload.extend(std::iter::repeat_n(b'[', depth));
626                payload.extend(std::iter::repeat_n(b']', depth));
627
628                let mut tokens = json_token_iter(&payload);
629                // We don't care about the exact result here (it will be a
630                // depth-limit error); we only care that the thread returns
631                // rather than aborting the process via stack overflow.
632                let _ = skip_value(&mut tokens);
633            })
634            .expect("failed to spawn small-stack thread");
635
636        handle
637            .join()
638            .expect("skip_value overflowed a 256KiB stack — recursion is unbounded");
639    }
640
641    /// Nesting below the depth limit should still succeed, demonstrating that
642    /// the iterative implementation preserves the happy path behaviour.
643    #[test]
644    fn skip_modestly_nested_still_works() {
645        // Below MAX_SKIP_DEPTH (256).
646        let depth = 100;
647        let mut payload = Vec::with_capacity(depth * 2 + 5);
648        payload.extend(std::iter::repeat_n(b'[', depth));
649        payload.extend(std::iter::repeat_n(b']', depth));
650        payload.extend_from_slice(b" true");
651
652        let mut tokens = json_token_iter(&payload);
653        skip_value(&mut tokens).expect("nesting below limit should parse");
654        assert!(matches!(
655            tokens.next(),
656            Some(Ok(Token::ValueBool { value: true, .. }))
657        ));
658    }
659
660    #[test]
661    fn test_expect_start_object() {
662        expect_err_custom(
663            "expected StartObject",
664            Some(2),
665            expect_start_object(value_bool(2, true)),
666        );
667        assert!(expect_start_object(start_object(0)).is_ok());
668    }
669
670    #[test]
671    fn test_expect_start_array() {
672        expect_err_custom(
673            "expected StartArray",
674            Some(2),
675            expect_start_array(value_bool(2, true)),
676        );
677        assert!(expect_start_array(start_array(0)).is_ok());
678    }
679
680    #[test]
681    fn test_expect_string_or_null() {
682        assert_eq!(None, expect_string_or_null(value_null(0)).unwrap());
683        assert_eq!(
684            Some(EscapedStr("test\\n")),
685            expect_string_or_null(value_string(0, "test\\n")).unwrap()
686        );
687        expect_err_custom(
688            "expected ValueString or ValueNull",
689            None,
690            expect_string_or_null(value_bool(0, true)),
691        );
692    }
693
694    #[test]
695    fn test_expect_number_or_null() {
696        assert_eq!(None, expect_number_or_null(value_null(0)).unwrap());
697        assert_eq!(
698            Some(Number::PosInt(5)),
699            expect_number_or_null(value_number(0, Number::PosInt(5))).unwrap()
700        );
701        expect_err_custom(
702            "expected ValueString, ValueNumber, or ValueNull",
703            None,
704            expect_number_or_null(value_bool(0, true)),
705        );
706        assert_eq!(
707            Some(Number::Float(f64::INFINITY)),
708            expect_number_or_null(value_string(0, "Infinity")).unwrap()
709        );
710        expect_err_custom(
711            "only `Infinity`, `-Infinity`, `NaN` can represent a float as a string but found `123`",
712            Some(0),
713            expect_number_or_null(value_string(0, "123")),
714        );
715        match expect_number_or_null(value_string(0, "NaN")) {
716            Ok(Some(Number::Float(v))) if v.is_nan() => {
717                // ok
718            }
719            not_ok => {
720                panic!("expected nan, found: {not_ok:?}")
721            }
722        }
723
724        // Test that infinity in ValueNumber token returns an error
725        let result = expect_number_or_null(value_number(0, Number::Float(f64::INFINITY)));
726        assert!(result.is_err(), "Expected error for infinity token");
727    }
728
729    #[test]
730    fn test_expect_blob_or_null() {
731        assert_eq!(None, expect_blob_or_null(value_null(0)).unwrap());
732        assert_eq!(
733            Some(Blob::new(b"hello!".to_vec())),
734            expect_blob_or_null(value_string(0, "aGVsbG8h")).unwrap()
735        );
736        expect_err_custom(
737            "expected ValueString or ValueNull",
738            None,
739            expect_blob_or_null(value_bool(0, true)),
740        );
741    }
742
743    #[test]
744    fn test_expect_timestamp_or_null() {
745        assert_eq!(
746            None,
747            expect_timestamp_or_null(value_null(0), Format::HttpDate).unwrap()
748        );
749        for (invalid, display_name) in &[
750            ("NaN", "NaN"),
751            ("Infinity", "infinity"),
752            ("-Infinity", "infinity"),
753        ] {
754            expect_err_custom(
755                format!("{display_name} is not a valid epoch").as_str(),
756                None,
757                expect_timestamp_or_null(value_string(0, invalid), Format::EpochSeconds),
758            );
759        }
760        assert_eq!(
761            Some(DateTime::from_secs_f64(2048.0)),
762            expect_timestamp_or_null(value_number(0, Number::Float(2048.0)), Format::EpochSeconds)
763                .unwrap()
764        );
765        assert_eq!(
766            Some(DateTime::from_secs_f64(1445412480.0)),
767            expect_timestamp_or_null(
768                value_string(0, "Wed, 21 Oct 2015 07:28:00 GMT"),
769                Format::HttpDate
770            )
771            .unwrap()
772        );
773        assert_eq!(
774            Some(DateTime::from_secs_f64(1445412480.0)),
775            expect_timestamp_or_null(value_string(0, "2015-10-21T07:28:00Z"), Format::DateTime)
776                .unwrap()
777        );
778        expect_err_custom(
779                "only `Infinity`, `-Infinity`, `NaN` can represent a float as a string but found `wrong`",
780                Some(0),
781            expect_timestamp_or_null(value_string(0, "wrong"), Format::EpochSeconds)
782        );
783        expect_err_custom(
784            "expected ValueString or ValueNull",
785            None,
786            expect_timestamp_or_null(value_number(0, Number::Float(0.0)), Format::DateTime),
787        );
788    }
789
790    #[test]
791    fn test_expect_document() {
792        let test = |value| expect_document(&mut json_token_iter(value).peekable()).unwrap();
793        assert_eq!(Document::Null, test(b"null"));
794        assert_eq!(Document::Bool(true), test(b"true"));
795        assert_eq!(Document::Number(Number::Float(3.2)), test(b"3.2"));
796        assert_eq!(Document::String("Foo\nBar".into()), test(b"\"Foo\\nBar\""));
797        assert_eq!(Document::Array(Vec::new()), test(b"[]"));
798        assert_eq!(Document::Object(HashMap::new()), test(b"{}"));
799        assert_eq!(
800            Document::Array(vec![
801                Document::Number(Number::PosInt(1)),
802                Document::Bool(false),
803                Document::String("s".into()),
804                Document::Array(Vec::new()),
805                Document::Object(HashMap::new()),
806            ]),
807            test(b"[1,false,\"s\",[],{}]")
808        );
809        assert_eq!(
810            Document::Object(
811                vec![
812                    ("num".to_string(), Document::Number(Number::PosInt(1))),
813                    ("bool".to_string(), Document::Bool(true)),
814                    ("string".to_string(), Document::String("s".into())),
815                    (
816                        "array".to_string(),
817                        Document::Array(vec![
818                            Document::Object(
819                                vec![("foo".to_string(), Document::Bool(false))]
820                                    .into_iter()
821                                    .collect(),
822                            ),
823                            Document::Object(
824                                vec![("bar".to_string(), Document::Bool(true))]
825                                    .into_iter()
826                                    .collect(),
827                            ),
828                        ])
829                    ),
830                    (
831                        "nested".to_string(),
832                        Document::Object(
833                            vec![("test".to_string(), Document::Null),]
834                                .into_iter()
835                                .collect()
836                        )
837                    ),
838                ]
839                .into_iter()
840                .collect()
841            ),
842            test(
843                br#"
844                { "num": 1,
845                  "bool": true,
846                  "string": "s",
847                  "array":
848                      [{ "foo": false },
849                       { "bar": true }],
850                  "nested": { "test": null } }
851                "#
852            )
853        );
854    }
855
856    #[test]
857    fn test_document_recursion_limit() {
858        let mut value = String::new();
859        value.extend(std::iter::repeat_n('[', 300));
860        value.extend(std::iter::repeat_n(']', 300));
861        expect_err_custom(
862            "exceeded max recursion depth while parsing document",
863            None,
864            expect_document(&mut json_token_iter(value.as_bytes()).peekable()),
865        );
866
867        value = String::new();
868        value.extend(std::iter::repeat_n("{\"t\":", 300));
869        value.push('1');
870        value.extend(std::iter::repeat_n('}', 300));
871        expect_err_custom(
872            "exceeded max recursion depth while parsing document",
873            None,
874            expect_document(&mut json_token_iter(value.as_bytes()).peekable()),
875        );
876    }
877
878    #[test]
879    fn test_expect_number_as_string_preserves_precision() {
880        use crate::deserialize::json_token_iter;
881
882        // Test large integer that fits in u64 but would lose precision in f64
883        // f64 has 53 bits of precision, so numbers > 2^53 lose precision
884        let input = b"18450000000000000000"; // 2^53 + 1, loses precision in f64
885        let mut iter = json_token_iter(input);
886        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
887        assert_eq!(result, Some("18450000000000000000"));
888
889        // Test large negative integer
890        let input = b"-9007199254740993";
891        let mut iter = json_token_iter(input);
892        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
893        assert_eq!(result, Some("-9007199254740993"));
894
895        // Test decimal with many digits
896        let input = b"123456789.123456789";
897        let mut iter = json_token_iter(input);
898        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
899        assert_eq!(result, Some("123456789.123456789"));
900
901        // Test scientific notation
902        let input = b"1.23e+50";
903        let mut iter = json_token_iter(input);
904        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
905        assert_eq!(result, Some("1.23e+50"));
906
907        // Test negative scientific notation
908        let input = b"-1.23e-50";
909        let mut iter = json_token_iter(input);
910        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
911        assert_eq!(result, Some("-1.23e-50"));
912
913        // Test null
914        let input = b"null";
915        let mut iter = json_token_iter(input);
916        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
917        assert_eq!(result, None);
918
919        // Test small numbers still work
920        let input = b"42";
921        let mut iter = json_token_iter(input);
922        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
923        assert_eq!(result, Some("42"));
924
925        // Test zero
926        let input = b"0";
927        let mut iter = json_token_iter(input);
928        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
929        assert_eq!(result, Some("0"));
930
931        // Test lowercase e in scientific notation is preserved
932        let input = b"2.5e-8";
933        let mut iter = json_token_iter(input);
934        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
935        assert_eq!(result, Some("2.5e-8"));
936
937        // Test uppercase E in scientific notation is preserved
938        let input = b"2.5E-8";
939        let mut iter = json_token_iter(input);
940        let result = expect_number_as_string_or_null(iter.next(), input).unwrap();
941        assert_eq!(result, Some("2.5E-8"));
942    }
943
944    #[test]
945    fn test_expect_number_as_string_error_cases() {
946        use crate::deserialize::json_token_iter;
947
948        // Test error when token is a string (not a number)
949        let input = b"\"not a number\"";
950        let mut iter = json_token_iter(input);
951        let result = expect_number_as_string_or_null(iter.next(), input);
952        assert!(result.is_err());
953
954        // Test error when token is a boolean
955        let input = b"true";
956        let mut iter = json_token_iter(input);
957        let result = expect_number_as_string_or_null(iter.next(), input);
958        assert!(result.is_err());
959
960        // Test error when token is an object
961        let input = b"{}";
962        let mut iter = json_token_iter(input);
963        let result = expect_number_as_string_or_null(iter.next(), input);
964        assert!(result.is_err());
965
966        // Test error when token is an array
967        let input = b"[]";
968        let mut iter = json_token_iter(input);
969        let result = expect_number_as_string_or_null(iter.next(), input);
970        assert!(result.is_err());
971    }
972
973    // Property-based tests to validate with random inputs
974    mod proptest_tests {
975        use super::*;
976        use crate::deserialize::json_token_iter;
977        use proptest::prelude::*;
978
979        proptest! {
980            #[test]
981            fn extracted_large_integer_matches_input(
982                // Generate 20-100 digit numbers (way bigger than i64 max: 19 digits)
983                num_str in "[1-9][0-9]{19,99}"
984            ) {
985                let input_bytes = num_str.as_bytes();
986                let mut iter = json_token_iter(input_bytes);
987                let result = expect_number_as_string_or_null(iter.next(), input_bytes)?;
988
989                prop_assert_eq!(result, Some(num_str.as_str()));
990            }
991
992            #[test]
993            fn extracted_large_negative_integer_matches_input(
994                // Generate negative numbers with 20-100 digits
995                num_str in "-[1-9][0-9]{19,99}"
996            ) {
997                let input_bytes = num_str.as_bytes();
998                let mut iter = json_token_iter(input_bytes);
999                let result = expect_number_as_string_or_null(iter.next(), input_bytes)?;
1000
1001                prop_assert_eq!(result, Some(num_str.as_str()));
1002            }
1003
1004            #[test]
1005            fn extracted_scientific_notation_matches_input(
1006                mantissa in -999999999i64..999999999i64,
1007                exponent in -100i32..100i32
1008            ) {
1009                let input = format!("{}e{}", mantissa, exponent);
1010                let input_bytes = input.as_bytes();
1011
1012                let mut iter = json_token_iter(input_bytes);
1013                let result = expect_number_as_string_or_null(iter.next(), input_bytes)?;
1014
1015                prop_assert_eq!(result, Some(input.as_str()));
1016            }
1017
1018            #[test]
1019            fn null_always_returns_none(
1020                // Generate random whitespace/formatting around null
1021                prefix in "[ \t\n\r]*",
1022                suffix in "[ \t\n\r]*"
1023            ) {
1024                let input = format!("{}null{}", prefix, suffix);
1025                let input_bytes = input.as_bytes();
1026
1027                let mut iter = json_token_iter(input_bytes);
1028                let result = expect_number_as_string_or_null(iter.next(), input_bytes)?;
1029
1030                prop_assert_eq!(result, None);
1031            }
1032        }
1033    }
1034}
aws_smithy_json/deserialize/token.rs

aws_smithy_json/deserialize/
token.rs
