move towards spannedtoken

src/error.rs

@@ -1,4 +1,4 @@
-use crate::lexer::Token;
+use crate::lexer::{SpannedToken, Token};
 use crate::num::ParseIntError;
 use crate::string::UnescapeError;
 use crate::Value;
@@ -162,29 +162,40 @@ impl Error for UnexpectedTokenError {}
 #[error("Invalid array key {0:?} expected number or string")]
 pub struct InvalidArrayKeyError(pub Value);
 
-pub trait ExpectToken {
+pub trait ExpectToken<'source> {
-    fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError>;
+    fn expect_token(
+        self,
+        expected: &[Token],
+    ) -> Result<SpannedToken<'source>, SpannedError<ParseError>>;
 }
 
-impl ExpectToken for Option<Token> {
+impl<'source> ExpectToken<'source> for Option<SpannedToken<'source>> {
-    fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError> {
+    fn expect_token(
+        self,
+        expected: &[Token],
+    ) -> Result<SpannedToken<'source>, SpannedError<ParseError>> {
         self.ok_or_else(|| UnexpectedTokenError {
             expected: expected.to_vec(),
             found: None,
         })
+        .with_span(usize::max_value()..usize::max_value())
         .and_then(|token| token.expect_token(expected))
     }
 }
 
-impl ExpectToken for Token {
+impl<'source> ExpectToken<'source> for SpannedToken<'source> {
-    fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError> {
+    fn expect_token(
-        if expected.iter().any(|expect| self.eq(expect)) {
+        self,
+        expected: &[Token],
+    ) -> Result<SpannedToken<'source>, SpannedError<ParseError>> {
+        if expected.iter().any(|expect| self.token.eq(expect)) {
             Ok(self)
         } else {
             Err(UnexpectedTokenError {
                 expected: expected.to_vec(),
-                found: Some(self),
+                found: Some(self.token),
             })
+            .with_span(self.span)
         }
     }
 }

src/lexer.rs

@@ -183,9 +183,17 @@ pub struct TokenStream<'source> {
 }
 
 impl<'source> TokenStream<'source> {
+    pub fn new(lexer: Lexer<'source, Token>) -> Self {
+        TokenStream { lexer }
+    }
+
     pub fn source(&self) -> &'source str {
         self.lexer.source()
     }
+
+    pub fn span(&self) -> Span {
+        self.lexer.span()
+    }
 }
 
 impl<'source> Iterator for TokenStream<'source> {

src/parser.rs

@@ -1,6 +1,6 @@
 use crate::error::UnexpectedTokenError;
 use crate::error::{ExpectToken, InvalidArrayKeyError, ParseError, ResultExt, SpannedError};
-use crate::lexer::Token;
+use crate::lexer::{SpannedToken, Token, TokenStream};
 use crate::num::parse_int;
 use crate::string::parse_string;
 use crate::{Key, Value};
@@ -27,53 +27,37 @@ use std::num::ParseFloatError;
 /// ```
 ///
 pub fn parse(source: &str) -> Result<Value, SpannedError<ParseError>> {
-    Parser::new(source).parse()
+    Parser::new(source).parse_any()
 }
 
 pub struct Parser<'source> {
     source: &'source str,
-    lexer: Lexer<'source, Token>,
+    tokens: TokenStream<'source>,
 }
 
 impl<'source> Parser<'source> {
     pub fn new(source: &'source str) -> Self {
         Parser {
             source,
-            lexer: Token::lexer(source),
+            tokens: TokenStream::new(Token::lexer(source)),
         }
     }
 
-    pub fn parse(&mut self) -> Result<Value, SpannedError<ParseError>> {
+    pub fn parse_any(&mut self) -> Result<Value, SpannedError<ParseError>> {
-        let token = self
+        let token = self.tokens.next().expect_token(&[
-            .lexer
+            Token::Bool,
-            .next()
+            Token::Integer,
-            .expect_token(&[
+            Token::Float,
-                Token::Bool,
+            Token::LiteralString,
-                Token::Integer,
+            Token::Null,
-                Token::Float,
+            Token::Array,
-                Token::LiteralString,
+            Token::SquareOpen,
-                Token::Null,
+        ])?;
-                Token::Array,
+        let value = match token.token {
-                Token::SquareOpen,
+            Token::Bool => Value::Bool(self.parse_bool(token)?),
-            ])
+            Token::Integer => Value::Int(self.parse_int(token)?),
-            .with_span(self.lexer.span())?;
+            Token::Float => Value::Float(self.parse_float(token)?),
-        let value = match token {
+            Token::LiteralString => Value::String(self.parse_string(token)?),
-            Token::Bool => Value::Bool(
-                self.lexer
-                    .slice()
-                    .to_ascii_lowercase()
-                    .parse()
-                    .with_span(self.lexer.span())?,
-            ),
-            Token::Integer => {
-                Value::Int(parse_int(self.lexer.slice()).with_span(self.lexer.span())?)
-            }
-            Token::Float => {
-                Value::Float(parse_float(self.lexer.slice()).with_span(self.lexer.span())?)
-            }
-            Token::LiteralString => {
-                Value::String(parse_string(self.lexer.slice()).with_span(self.lexer.span())?)
-            }
             Token::Null => Value::Null,
             Token::Array => Value::Array(self.parse_array(ArraySyntax::Long)?),
             Token::SquareOpen => Value::Array(self.parse_array(ArraySyntax::Short)?),
@@ -83,6 +67,26 @@ impl<'source> Parser<'source> {
         Ok(value)
     }
 
+    fn parse_bool(&self, token: SpannedToken) -> Result<bool, SpannedError<ParseError>> {
+        token
+            .slice()
+            .to_ascii_lowercase()
+            .parse()
+            .with_span(token.span)
+    }
+
+    fn parse_int(&self, token: SpannedToken) -> Result<i64, SpannedError<ParseError>> {
+        parse_int(token.slice()).with_span(token.span)
+    }
+
+    fn parse_float(&self, token: SpannedToken) -> Result<f64, SpannedError<ParseError>> {
+        parse_float(token.slice()).with_span(token.span)
+    }
+
+    fn parse_string(&self, token: SpannedToken) -> Result<String, SpannedError<ParseError>> {
+        parse_string(token.slice()).with_span(token.span)
+    }
+
     fn parse_array(
         &mut self,
         syntax: ArraySyntax,
@@ -90,14 +94,11 @@ impl<'source> Parser<'source> {
         let mut builder = ArrayBuilder::default();
 
         if syntax == ArraySyntax::Long {
-            self.lexer
+            self.tokens.next().expect_token(&[Token::BracketOpen])?;
-                .next()
-                .expect_token(&[Token::BracketOpen])
-                .with_span(self.lexer.span())?;
         }
 
         loop {
-            let key_or_value = match self.parse() {
+            let key_or_value = match self.parse_any() {
                 Ok(value) => value,
                 Err(err) => {
                     // trailing comma or empty array
@@ -110,14 +111,14 @@ impl<'source> Parser<'source> {
                     }
                 }
             };
-            let key_or_value_span = self.lexer.span();
+            let key_or_value_span = self.tokens.span();
-            let next = self
+            let next = self.tokens.next().expect_token(&[
-                .lexer
+                syntax.close_bracket(),
-                .next()
+                Token::Comma,
-                .expect_token(&[syntax.close_bracket(), Token::Comma, Token::Arrow])
+                Token::Arrow,
-                .with_span(self.lexer.span())?;
+            ])?;
 
-            match next {
+            match next.token {
                 Token::BracketClose => {
                     builder.push_value(key_or_value);
                     break;
@@ -130,7 +131,7 @@ impl<'source> Parser<'source> {
                     builder.push_value(key_or_value);
                 }
                 Token::Arrow => {
-                    let value = self.parse()?;
+                    let value = self.parse_any()?;
                     let key = match key_or_value {
                         Value::Int(int) => Key::Int(int),
                         Value::Float(float) => Key::Int(float as i64),
@@ -144,10 +145,10 @@ impl<'source> Parser<'source> {
                     builder.push_key_value(key, value);
 
                     match self
-                        .lexer
+                        .tokens
                         .next()
-                        .expect_token(&[syntax.close_bracket(), Token::Comma])
+                        .expect_token(&[syntax.close_bracket(), Token::Comma])?
-                        .with_span(self.lexer.span())?
+                        .token
                     {
                         Token::BracketClose => {
                             break;