aws_smithy_json/
deserialize.rs

Help
1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use crate::deserialize::error::{DeserializeError as Error, DeserializeErrorKind as ErrorKind};
7use aws_smithy_types::Number;
8use ErrorKind::*;
9
10pub mod error;
11pub mod token;
12
13pub use token::{EscapeError, EscapedStr, Offset, Token};
14
15/// JSON token parser as a Rust iterator
16///
17/// This parser will parse and yield exactly one [`Token`] per iterator `next()` call.
18/// Validation is done on the fly, so it is possible for it to parse an invalid JSON document
19/// until it gets to the first [`Error`].
20///
21/// JSON string values are left escaped in the [`Token::ValueString`] as an [`EscapedStr`],
22/// which is a new type around a slice of original `input` bytes so that the caller can decide
23/// when to unescape and allocate into a [`String`].
24///
25/// The parser *will* accept multiple valid JSON values. For example, `b"null true"` will
26/// yield `ValueNull` and `ValueTrue`. It is the responsibility of the caller to handle this for
27/// their use-case.
28pub fn json_token_iter(input: &[u8]) -> JsonTokenIterator<'_> {
29    JsonTokenIterator {
30        input,
31        index: 0,
32        state_stack: vec![State::Initial],
33    }
34}
35
36/// Internal parser state for the iterator. Used to context between successive `next` calls.
37#[derive(Copy, Clone, Debug, Eq, PartialEq)]
38enum State {
39    /// Entry point. Expecting any JSON value.
40    Initial,
41    /// Expecting the next token to be the *first* value in an array, or the end of the array.
42    ArrayFirstValueOrEnd,
43    /// Expecting the next token to the next value in an array, or the end of the array.
44    ArrayNextValueOrEnd,
45    /// Expecting the next token to be the *first* key in the object, or the end of the object.
46    ObjectFirstKeyOrEnd,
47    /// Expecting the next token to the next object key, or the end of the object.
48    ObjectNextKeyOrEnd,
49    /// Expecting the next token to be the value of a field in an object.
50    ObjectFieldValue,
51}
52
53/// An iterator over a `&[u8]` that yields `Result<Token, Error>` with [Token] being JSON tokens.
54/// Construct with [json_token_iter].
55pub struct JsonTokenIterator<'a> {
56    input: &'a [u8],
57    index: usize,
58    state_stack: Vec<State>,
59}
60
61impl<'a> JsonTokenIterator<'a> {
62    /// Previews the next byte.
63    fn peek_byte(&self) -> Option<u8> {
64        if self.index >= self.input.len() {
65            None
66        } else {
67            Some(self.input[self.index])
68        }
69    }
70
71    /// Expects there to be another byte coming up, and previews it.
72    /// If there isn't, an `UnexpectedEOS` error is returned.
73    fn peek_expect(&self) -> Result<u8, Error> {
74        self.peek_byte().ok_or_else(|| self.error(UnexpectedEos))
75    }
76
77    /// Advances to the next byte in the stream.
78    fn advance(&mut self) {
79        if self.index < self.input.len() {
80            self.index += 1;
81        }
82    }
83
84    /// Advances and returns the next byte in the stream.
85    fn next_byte(&mut self) -> Option<u8> {
86        let next = self.peek_byte();
87        self.advance();
88        next
89    }
90
91    /// Expects there to be another byte coming up, and returns it while advancing.
92    /// If there isn't, an `UnexpectedEOS` error is returned.
93    fn next_expect(&mut self) -> Result<u8, Error> {
94        self.next_byte().ok_or_else(|| self.error(UnexpectedEos))
95    }
96
97    /// Creates an error at the given `offset` in the stream.
98    fn error_at(&self, offset: usize, kind: ErrorKind) -> Error {
99        Error::new(kind, Some(offset))
100    }
101
102    /// Creates an error at the current offset in the stream.
103    fn error(&self, kind: ErrorKind) -> Error {
104        self.error_at(self.index, kind)
105    }
106
107    /// Advances until it hits a non-whitespace character or the end of the slice.
108    fn discard_whitespace(&mut self) {
109        while let Some(byte) = self.peek_byte() {
110            match byte {
111                b' ' | b'\t' | b'\r' | b'\n' => {
112                    self.advance();
113                }
114                _ => break,
115            }
116        }
117    }
118
119    /// Returns the top of the state stack (current state).
120    fn state(&self) -> State {
121        self.state_stack[self.state_stack.len() - 1]
122    }
123
124    /// Replaces the top of the state stack with a new `state`.
125    fn replace_state(&mut self, state: State) {
126        self.state_stack.pop();
127        self.state_stack.push(state);
128    }
129
130    /// Returns current offset
131    fn offset(&self) -> Offset {
132        Offset(self.index)
133    }
134
135    /// Discards the '{' character and pushes the `ObjectFirstKeyOrEnd` state.
136    fn start_object(&mut self) -> Token<'a> {
137        let offset = self.offset();
138        let byte = self.next_byte();
139        debug_assert_eq!(byte, Some(b'{'));
140        self.state_stack.push(State::ObjectFirstKeyOrEnd);
141        Token::StartObject { offset }
142    }
143
144    /// Discards the '}' character and pops the current state.
145    fn end_object(&mut self) -> Token<'a> {
146        let offset = self.offset();
147        let (byte, state) = (self.next_byte(), self.state_stack.pop());
148        debug_assert_eq!(byte, Some(b'}'));
149        debug_assert!(
150            state == Some(State::ObjectFirstKeyOrEnd) || state == Some(State::ObjectNextKeyOrEnd)
151        );
152        Token::EndObject { offset }
153    }
154
155    /// Discards the '[' character and pushes the `ArrayFirstValueOrEnd` state.
156    fn start_array(&mut self) -> Token<'a> {
157        let offset = self.offset();
158        let byte = self.next_byte();
159        debug_assert_eq!(byte, Some(b'['));
160        self.state_stack.push(State::ArrayFirstValueOrEnd);
161        Token::StartArray { offset }
162    }
163
164    /// Discards the ']' character and pops the current state.
165    fn end_array(&mut self) -> Token<'a> {
166        let offset = self.offset();
167        let (byte, state) = (self.next_byte(), self.state_stack.pop());
168        debug_assert_eq!(byte, Some(b']'));
169        debug_assert!(
170            state == Some(State::ArrayFirstValueOrEnd) || state == Some(State::ArrayNextValueOrEnd)
171        );
172        Token::EndArray { offset }
173    }
174
175    /// Reads a JSON string out of the stream.
176    fn read_string(&mut self) -> Result<&'a str, Error> {
177        // Skip the starting quote
178        let quote_byte = self.next_byte();
179        debug_assert_eq!(quote_byte, Some(b'\"'));
180
181        // Read bytes until a non-escaped end-quote, unescaping sequences as needed on the fly
182        let start = self.index;
183        loop {
184            match self.peek_expect()? {
185                b'"' => {
186                    let value = std::str::from_utf8(&self.input[start..self.index])
187                        .map_err(|_| self.error(InvalidUtf8))?;
188                    self.advance();
189                    return Ok(value);
190                }
191                b'\\' => match self.next_expect()? {
192                    b'\\' | b'/' | b'"' | b'b' | b'f' | b'n' | b'r' | b't' => self.advance(),
193                    b'u' => {
194                        if self.index + 4 > self.input.len() {
195                            return Err(self.error_at(self.input.len(), UnexpectedEos));
196                        }
197                        self.index += 4;
198                    }
199                    byte => return Err(self.error(InvalidEscape(byte.into()))),
200                },
201                byte @ 0x00..=0x1F => return Err(self.error(UnexpectedControlCharacter(byte))),
202                _ => self.advance(),
203            }
204        }
205    }
206
207    /// Expects the given literal to be next in the stream.
208    fn expect_literal(&mut self, expected: &[u8]) -> Result<(), Error> {
209        let (start, end) = (self.index, self.index + expected.len());
210        if end > self.input.len() {
211            return Err(self.error_at(self.input.len(), UnexpectedEos));
212        }
213        if expected != &self.input[start..end] {
214            return Err(self.error_at(
215                start,
216                ExpectedLiteral(std::str::from_utf8(expected).unwrap().into()),
217            ));
218        }
219        self.index = end;
220        Ok(())
221    }
222
223    /// Expects a literal `null` next in the stream.
224    fn expect_null(&mut self) -> Result<Token<'a>, Error> {
225        let offset = self.offset();
226        self.expect_literal(b"null")?;
227        Ok(Token::ValueNull { offset })
228    }
229
230    /// Expects a boolean `true` / `false` to be next in the stream and returns its value.
231    fn expect_bool(&mut self) -> Result<Token<'a>, Error> {
232        let offset = self.offset();
233        match self.peek_expect()? {
234            b't' => {
235                self.expect_literal(b"true")?;
236                Ok(Token::ValueBool {
237                    offset,
238                    value: true,
239                })
240            }
241            b'f' => {
242                self.expect_literal(b"false")?;
243                Ok(Token::ValueBool {
244                    offset,
245                    value: false,
246                })
247            }
248            _ => unreachable!(
249                "this function must only be called when the next character is 't' or 'f'"
250            ),
251        }
252    }
253
254    /// Advances passed the exponent part of a floating point number.
255    fn skip_exponent(&mut self) {
256        self.advance();
257        match self.peek_byte() {
258            Some(b'-') => self.advance(),
259            Some(b'+') => self.advance(),
260            _ => {}
261        }
262        while let Some(b'0'..=b'9') = self.peek_byte() {
263            self.advance();
264        }
265    }
266
267    /// Advances passed the decimal part of a floating point number.
268    fn skip_decimal(&mut self) {
269        self.advance();
270        while let Some(byte) = self.peek_byte() {
271            match byte {
272                b'0'..=b'9' => self.advance(),
273                b'e' | b'E' => self.skip_exponent(),
274                _ => break,
275            }
276        }
277    }
278
279    /// Starting from the current location in the stream, this advances until
280    /// it finds a character that doesn't look like its part of a number, and then
281    /// returns `(start_index, end_index, negative, floating)`, with `start_index`
282    /// and `end_index` representing the slice of the stream that is the number,
283    /// `negative` whether or not it is a negative number, and `floating` whether or not
284    /// the number contains a decimal point and/or an exponent.
285    fn scan_number(&mut self) -> (usize, usize, bool, bool) {
286        let start_index = self.index;
287        let negative = if self.peek_byte() == Some(b'-') {
288            self.advance();
289            true
290        } else {
291            false
292        };
293        let mut floating = false;
294        while let Some(byte) = self.peek_byte() {
295            match byte {
296                b'0'..=b'9' => self.advance(),
297                b'.' => {
298                    floating = true;
299                    self.skip_decimal();
300                }
301                b'e' | b'E' => {
302                    floating = true;
303                    self.skip_exponent();
304                }
305                _ => break,
306            }
307        }
308        (start_index, self.index, negative, floating)
309    }
310
311    /// Expects a number in the stream, and returns its value.
312    fn expect_number(&mut self) -> Result<Token<'a>, Error> {
313        let offset = self.offset();
314        let (start, end, negative, floating) = self.scan_number();
315        let number_slice = &self.input[start..end];
316
317        // Unsafe: we examined every character in the range, and they are all number characters
318        debug_assert!(std::str::from_utf8(number_slice).is_ok());
319        let number_str = unsafe { std::str::from_utf8_unchecked(number_slice) };
320
321        use std::str::FromStr;
322        Ok(Token::ValueNumber {
323            offset,
324            value: if floating {
325                Number::Float(
326                    f64::from_str(number_str)
327                        .map_err(|_| self.error_at(start, InvalidNumber))
328                        .and_then(|f| {
329                            must_be_finite(f).map_err(|_| self.error_at(start, InvalidNumber))
330                        })?,
331                )
332            } else if negative {
333                // If the negative value overflows, then stuff it into an f64
334                let positive = u64::from_str(&number_str[1..])
335                    .map_err(|_| self.error_at(start, InvalidNumber))?;
336                let negative = positive.wrapping_neg() as i64;
337                if negative > 0 {
338                    Number::Float(-(positive as f64))
339                } else {
340                    Number::NegInt(negative)
341                }
342            } else {
343                Number::PosInt(
344                    u64::from_str(number_str).map_err(|_| self.error_at(start, InvalidNumber))?,
345                )
346            },
347        })
348    }
349
350    /// Reads a value from the stream and returns the next token. For objects and arrays,
351    /// the entire object or array will not be ready, but rather, a [Token::StartObject]/[Token::StartArray]
352    /// will be returned.
353    fn read_value(&mut self) -> Result<Token<'a>, Error> {
354        self.discard_whitespace();
355        let offset = self.offset();
356        match self.peek_expect()? {
357            b'{' => Ok(self.start_object()),
358            b'[' => Ok(self.start_array()),
359            b'"' => self.read_string().map(|s| Token::ValueString {
360                offset,
361                value: EscapedStr::new(s),
362            }),
363            byte => {
364                let value = match byte {
365                    b'n' => self.expect_null(),
366                    b't' | b'f' => self.expect_bool(),
367                    b'-' | (b'0'..=b'9') => self.expect_number(),
368                    byte => Err(self.error(UnexpectedToken(
369                        byte.into(),
370                        "'{', '[', '\"', 'null', 'true', 'false', <number>",
371                    ))),
372                }?;
373                // Verify there are no unexpected trailers on the end of the value
374                if let Some(byte) = self.peek_byte() {
375                    match byte {
376                        b' ' | b'\t' | b'\r' | b'\n' | b'}' | b']' | b',' => {}
377                        _ => {
378                            return Err(self.error(UnexpectedToken(
379                                byte.into(),
380                                "<whitespace>, '}', ']', ','",
381                            )))
382                        }
383                    }
384                }
385                Ok(value)
386            }
387        }
388    }
389
390    /// Handles the [State::ArrayFirstValueOrEnd] state.
391    fn state_array_first_value_or_end(&mut self) -> Result<Token<'a>, Error> {
392        match self.peek_expect()? {
393            b']' => Ok(self.end_array()),
394            _ => {
395                self.replace_state(State::ArrayNextValueOrEnd);
396                self.read_value()
397            }
398        }
399    }
400
401    /// Handles the [State::ArrayNextValueOrEnd] state.
402    fn state_array_next_value_or_end(&mut self) -> Result<Token<'a>, Error> {
403        match self.peek_expect()? {
404            b']' => Ok(self.end_array()),
405            b',' => {
406                self.advance();
407                self.read_value()
408            }
409            byte => Err(self.error(UnexpectedToken(byte.into(), "']', ','"))),
410        }
411    }
412
413    /// Expects an object key.
414    fn object_key(&mut self) -> Result<Token<'a>, Error> {
415        let offset = self.offset();
416        match self.peek_expect()? {
417            b'"' => {
418                self.replace_state(State::ObjectFieldValue);
419                self.read_string().map(|s| Token::ObjectKey {
420                    offset,
421                    key: EscapedStr::new(s),
422                })
423            }
424            byte => Err(self.error(UnexpectedToken(byte.into(), "'\"'"))),
425        }
426    }
427
428    /// Handles the [State::ObjectFirstKeyOrEnd] state.
429    fn state_object_first_key_or_end(&mut self) -> Result<Token<'a>, Error> {
430        match self.peek_expect()? {
431            b'}' => Ok(self.end_object()),
432            _ => self.object_key(),
433        }
434    }
435
436    /// Handles the [State::ObjectNextKeyOrEnd] state.
437    fn state_object_next_key_or_end(&mut self) -> Result<Token<'a>, Error> {
438        match self.peek_expect()? {
439            b'}' => Ok(self.end_object()),
440            b',' => {
441                self.advance();
442                self.discard_whitespace();
443                self.object_key()
444            }
445            byte => Err(self.error(UnexpectedToken(byte.into(), "'}', ','"))),
446        }
447    }
448
449    /// Handles the [State::ObjectFieldValue] state.
450    fn state_object_field_value(&mut self) -> Result<Token<'a>, Error> {
451        match self.peek_expect()? {
452            b':' => {
453                self.advance();
454                self.replace_state(State::ObjectNextKeyOrEnd);
455                self.read_value()
456            }
457            byte => Err(self.error(UnexpectedToken(byte.into(), "':'"))),
458        }
459    }
460}
461
462impl<'a> Iterator for JsonTokenIterator<'a> {
463    type Item = Result<Token<'a>, Error>;
464
465    fn next(&mut self) -> Option<Self::Item> {
466        debug_assert!(self.index <= self.input.len());
467        if self.index == self.input.len() {
468            return None;
469        }
470
471        self.discard_whitespace();
472        let result = match self.state() {
473            State::Initial => self.peek_byte().map(|_| self.read_value()),
474            State::ArrayFirstValueOrEnd => Some(self.state_array_first_value_or_end()),
475            State::ArrayNextValueOrEnd => Some(self.state_array_next_value_or_end()),
476            State::ObjectFirstKeyOrEnd => Some(self.state_object_first_key_or_end()),
477            State::ObjectNextKeyOrEnd => Some(self.state_object_next_key_or_end()),
478            State::ObjectFieldValue => Some(self.state_object_field_value()),
479        };
480        // Invalidate the stream if we encountered an error
481        if result.as_ref().map(|r| r.is_err()).unwrap_or(false) {
482            self.index = self.input.len();
483        }
484        result
485    }
486}
487
488fn must_be_finite(f: f64) -> Result<f64, ()> {
489    if f.is_finite() {
490        Ok(f)
491    } else {
492        Err(())
493    }
494}
495
496fn must_not_be_finite(f: f64) -> Result<f64, ()> {
497    if !f.is_finite() {
498        Ok(f)
499    } else {
500        Err(())
501    }
502}
503
504#[cfg(test)]
505mod tests {
506    use crate::deserialize::error::{DeserializeError as Error, DeserializeErrorKind as ErrorKind};
507    use crate::deserialize::token::test::{
508        end_array, end_object, object_key, start_array, start_object, value_bool, value_null,
509        value_number, value_string,
510    };
511    use crate::deserialize::{json_token_iter, EscapedStr, Token};
512    use aws_smithy_types::Number;
513    use proptest::prelude::*;
514
515    #[track_caller]
516    fn expect_token(
517        expected: Option<Result<Token<'_>, Error>>,
518        actual: Option<Result<Token<'_>, Error>>,
519    ) {
520        let (expected, actual) = (
521            expected.transpose().expect("err in expected"),
522            actual.transpose().expect("err in actual"),
523        );
524        assert_eq!(expected, actual);
525    }
526
527    macro_rules! expect_err {
528        ($kind:pat, $offset:expr, $value:expr) => {
529            let err: Error = $value.transpose().err().expect("expected error");
530            assert!(matches!(err.kind, $kind));
531            assert_eq!($offset, err.offset);
532        };
533    }
534
535    #[test]
536    fn test_empty() {
537        assert!(json_token_iter(b"").next().is_none());
538        assert!(json_token_iter(b" ").next().is_none());
539        assert!(json_token_iter(b"\t").next().is_none());
540    }
541
542    #[test]
543    fn test_empty_string() {
544        let mut iter = json_token_iter(b"\"\"");
545        expect_token(value_string(0, ""), iter.next());
546        expect_token(None, iter.next());
547
548        let mut iter = json_token_iter(b" \r\n\t \"\"  ");
549        expect_token(value_string(5, ""), iter.next());
550        expect_token(None, iter.next());
551    }
552
553    #[test]
554    fn test_empty_array() {
555        let mut iter = json_token_iter(b"[]");
556        expect_token(start_array(0), iter.next());
557        expect_token(end_array(1), iter.next());
558        expect_token(None, iter.next());
559    }
560
561    #[test]
562    fn test_empty_object() {
563        let mut iter = json_token_iter(b"{}");
564        expect_token(start_object(0), iter.next());
565        expect_token(end_object(1), iter.next());
566        expect_token(None, iter.next());
567    }
568
569    #[test]
570    fn test_null() {
571        expect_token(value_null(1), json_token_iter(b" null ").next());
572
573        let mut iter = json_token_iter(b"[null, null,null]");
574        expect_token(start_array(0), iter.next());
575        expect_token(value_null(1), iter.next());
576        expect_token(value_null(7), iter.next());
577        expect_token(value_null(12), iter.next());
578        expect_token(end_array(16), iter.next());
579        expect_token(None, iter.next());
580
581        assert!(json_token_iter(b"n").next().unwrap().is_err());
582        assert!(json_token_iter(b"nul").next().unwrap().is_err());
583        assert!(json_token_iter(b"nulll").next().unwrap().is_err());
584    }
585
586    #[test]
587    fn test_bools() {
588        assert!(json_token_iter(b"tru").next().unwrap().is_err());
589        assert!(json_token_iter(b"truee").next().unwrap().is_err());
590        assert!(json_token_iter(b"f").next().unwrap().is_err());
591        assert!(json_token_iter(b"falsee").next().unwrap().is_err());
592        expect_token(value_bool(1, true), json_token_iter(b" true ").next());
593        expect_token(value_bool(0, false), json_token_iter(b"false").next());
594
595        let mut iter = json_token_iter(b"[true,false]");
596        expect_token(start_array(0), iter.next());
597        expect_token(value_bool(1, true), iter.next());
598        expect_token(value_bool(6, false), iter.next());
599        expect_token(end_array(11), iter.next());
600        expect_token(None, iter.next());
601    }
602
603    proptest! {
604        #[test]
605        fn string_prop_test(input in ".*") {
606            let json: String = serde_json::to_string(&input).unwrap();
607            let mut iter = json_token_iter(json.as_bytes());
608            expect_token(value_string(0, &json[1..(json.len() - 1)]), iter.next());
609            expect_token(None, iter.next());
610        }
611
612        #[test]
613        fn integer_prop_test(input: i64) {
614            let json = serde_json::to_string(&input).unwrap();
615            let mut iter = json_token_iter(json.as_bytes());
616            let expected = if input < 0 {
617                Number::NegInt(input)
618            } else {
619                Number::PosInt(input as u64)
620            };
621            expect_token(value_number(0, expected), iter.next());
622            expect_token(None, iter.next());
623        }
624
625        #[test]
626        fn float_prop_test(input: f64) {
627            let json = serde_json::to_string(&input).unwrap();
628            let mut iter = json_token_iter(json.as_bytes());
629            expect_token(value_number(0, Number::Float(input)), iter.next());
630            expect_token(None, iter.next());
631        }
632    }
633
634    #[test]
635    fn valid_numbers() {
636        let expect = |number, input| {
637            expect_token(value_number(0, number), json_token_iter(input).next());
638        };
639        expect(Number::Float(0.0), b"0.");
640        expect(Number::Float(0.0), b"0e0");
641        expect(Number::Float(0.0), b"0E0");
642        expect(Number::Float(10.0), b"1E1");
643        expect(Number::Float(10.0), b"1E+1");
644        expect(Number::Float(100.0), b"1e+2");
645
646        expect(Number::NegInt(-50000), b"-50000");
647        expect(
648            Number::Float(-18446744073709551615.0),
649            b"-18446744073709551615",
650        );
651    }
652
653    // These cases actually shouldn't parse according to the spec, but it's easier
654    // to be lenient on these, and it doesn't really impact the SDK use-case.
655    #[test]
656    fn invalid_numbers_we_are_intentionally_accepting() {
657        let expect = |number, input| {
658            expect_token(value_number(0, number), json_token_iter(input).next());
659        };
660
661        expect(Number::NegInt(-1), b"-01");
662        expect(Number::Float(-2.0), b"-2.");
663        expect(Number::Float(0.0), b"0.e1");
664        expect(Number::Float(0.002), b"2.e-3");
665        expect(Number::Float(2000.0), b"2.e3");
666        expect(Number::NegInt(-12), b"-012");
667        expect(Number::Float(-0.123), b"-.123");
668        expect(Number::Float(1.0), b"1.");
669        expect(Number::PosInt(12), b"012");
670    }
671
672    #[test]
673    fn invalid_numbers() {
674        macro_rules! unexpected_token {
675            ($input:expr, $token:pat, $offset:expr, $msg:pat) => {
676                let tokens: Vec<Result<Token<'_>, Error>> = json_token_iter($input).collect();
677                assert_eq!(1, tokens.len());
678                expect_err!(
679                    ErrorKind::UnexpectedToken($token, $msg),
680                    Some($offset),
681                    tokens.into_iter().next()
682                );
683            };
684        }
685
686        let invalid_number = |input, offset| {
687            let tokens: Vec<Result<Token<'_>, Error>> = json_token_iter(input).collect();
688            assert_eq!(1, tokens.len());
689            expect_err!(
690                ErrorKind::InvalidNumber,
691                Some(offset),
692                tokens.into_iter().next()
693            );
694        };
695
696        unexpected_token!(
697            b".",
698            '.',
699            0,
700            "'{', '[', '\"', 'null', 'true', 'false', <number>"
701        );
702        unexpected_token!(
703            b".0",
704            '.',
705            0,
706            "'{', '[', '\"', 'null', 'true', 'false', <number>"
707        );
708        unexpected_token!(b"0-05", '-', 1, "<whitespace>, '}', ']', ','");
709        unexpected_token!(b"0x05", 'x', 1, "<whitespace>, '}', ']', ','");
710        unexpected_token!(b"123.invalid", 'i', 4, "<whitespace>, '}', ']', ','");
711        unexpected_token!(b"123invalid", 'i', 3, "<whitespace>, '}', ']', ','");
712        unexpected_token!(
713            b"asdf",
714            'a',
715            0,
716            "'{', '[', '\"', 'null', 'true', 'false', <number>"
717        );
718
719        invalid_number(b"-a", 0);
720        invalid_number(b"1e", 0);
721        invalid_number(b"1e-", 0);
722
723        // Number parsing fails before it even looks at the trailer because of invalid exponent
724        invalid_number(b"123.0Einvalid", 0);
725    }
726
727    #[test]
728    fn test_unclosed_array() {
729        let mut iter = json_token_iter(br#" [null "#);
730        expect_token(start_array(1), iter.next());
731        expect_token(value_null(2), iter.next());
732        expect_err!(ErrorKind::UnexpectedEos, Some(7), iter.next());
733    }
734
735    #[test]
736    fn test_array_with_items() {
737        let mut iter = json_token_iter(b"[[], {}, \"test\"]");
738        expect_token(start_array(0), iter.next());
739        expect_token(start_array(1), iter.next());
740        expect_token(end_array(2), iter.next());
741        expect_token(start_object(5), iter.next());
742        expect_token(end_object(6), iter.next());
743        expect_token(value_string(9, "test"), iter.next());
744        expect_token(end_array(15), iter.next());
745        expect_token(None, iter.next());
746    }
747
748    #[test]
749    fn test_object_with_items() {
750        let mut tokens = json_token_iter(
751            br#"{ "some_int": 5,
752                  "some_float": 5.2,
753                  "some_negative": -5,
754                  "some_negative_float": -2.4,
755                  "some_string": "test",
756                  "some_struct": { "nested": "asdf" },
757                  "some_array": ["one", "two"] }"#,
758        );
759        expect_token(start_object(0), tokens.next());
760        expect_token(object_key(2, "some_int"), tokens.next());
761        expect_token(value_number(14, Number::PosInt(5)), tokens.next());
762        expect_token(object_key(35, "some_float"), tokens.next());
763        expect_token(value_number(49, Number::Float(5.2)), tokens.next());
764        expect_token(object_key(72, "some_negative"), tokens.next());
765        expect_token(value_number(89, Number::NegInt(-5)), tokens.next());
766        expect_token(object_key(111, "some_negative_float"), tokens.next());
767        expect_token(value_number(134, Number::Float(-2.4)), tokens.next());
768        expect_token(object_key(158, "some_string"), tokens.next());
769        expect_token(value_string(173, "test"), tokens.next());
770        expect_token(object_key(199, "some_struct"), tokens.next());
771        expect_token(start_object(214), tokens.next());
772        expect_token(object_key(216, "nested"), tokens.next());
773        expect_token(value_string(226, "asdf"), tokens.next());
774        expect_token(end_object(233), tokens.next());
775        expect_token(object_key(254, "some_array"), tokens.next());
776        expect_token(start_array(268), tokens.next());
777        expect_token(value_string(269, "one"), tokens.next());
778        expect_token(value_string(276, "two"), tokens.next());
779        expect_token(end_array(281), tokens.next());
780        expect_token(end_object(283), tokens.next());
781        expect_token(None, tokens.next());
782    }
783
784    #[test]
785    fn test_object_trailing_comma() {
786        let mut iter = json_token_iter(br#" { "test": "trailing", } "#);
787        expect_token(start_object(1), iter.next());
788        expect_token(object_key(3, "test"), iter.next());
789        expect_token(value_string(11, "trailing"), iter.next());
790        expect_err!(
791            ErrorKind::UnexpectedToken('}', "'\"'"),
792            Some(23),
793            iter.next()
794        );
795        assert!(iter.next().is_none());
796    }
797
798    #[test]
799    fn test_object_no_colon() {
800        let mut iter = json_token_iter(br#" {"test" "#);
801        expect_token(start_object(1), iter.next());
802        expect_token(object_key(2, "test"), iter.next());
803        expect_err!(ErrorKind::UnexpectedEos, Some(9), iter.next());
804        expect_token(None, iter.next());
805    }
806
807    #[test]
808    fn unescaped_ctrl_characters() {
809        assert!(json_token_iter(b"\"test\x00test\"")
810            .next()
811            .unwrap()
812            .is_err());
813        assert!(json_token_iter(b"\"test\ntest\"").next().unwrap().is_err());
814        assert!(json_token_iter(b"\"test\ttest\"").next().unwrap().is_err());
815    }
816
817    #[test]
818    fn escaped_str() {
819        let escaped = EscapedStr::new("foo\\nbar");
820        assert_eq!("foo\\nbar", escaped.as_escaped_str());
821        assert_eq!("foo\nbar", escaped.to_unescaped().unwrap());
822    }
823}
aws_smithy_json/deserialize.rs

aws_smithy_json/
deserialize.rs
