robin/php-literal-parser
1
0
Fork 0
mirror of https://codeberg.org/icewind/php-literal-parser.git synced 2026-06-03 18:44:07 +02:00
Code Issues Projects Releases Wiki Activity

move towards spannedtoken

Browse source
This commit is contained in:
Robin Appelman 2020-12-13 17:24:28 +01:00
commit 33bc8f160b
3 changed files with 80 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/error.rs
View file

@ -1,4 +1,4 @@
use crate::lexer::Token;
use crate::lexer::{SpannedToken, Token};
use crate::num::ParseIntError;
use crate::string::UnescapeError;
use crate::Value;
@ -162,29 +162,40 @@ impl Error for UnexpectedTokenError {}
#[error("Invalid array key {0:?} expected number or string")]
pub struct InvalidArrayKeyError(pub Value);
pub trait ExpectToken {
fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError>;
pub trait ExpectToken<'source> {
fn expect_token(
self,
expected: &[Token],
) -> Result<SpannedToken<'source>, SpannedError<ParseError>>;
}
impl ExpectToken for Option<Token> {
fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError> {
impl<'source> ExpectToken<'source> for Option<SpannedToken<'source>> {
fn expect_token(
self,
expected: &[Token],
) -> Result<SpannedToken<'source>, SpannedError<ParseError>> {
self.ok_or_else(|| UnexpectedTokenError {
expected: expected.to_vec(),
found: None,
})
.with_span(usize::max_value()..usize::max_value())
.and_then(|token| token.expect_token(expected))
}
}
impl ExpectToken for Token {
fn expect_token(self, expected: &[Token]) -> Result<Token, UnexpectedTokenError> {
if expected.iter().any(|expect| self.eq(expect)) {
impl<'source> ExpectToken<'source> for SpannedToken<'source> {
fn expect_token(
self,
expected: &[Token],
) -> Result<SpannedToken<'source>, SpannedError<ParseError>> {
if expected.iter().any(|expect| self.token.eq(expect)) {
Ok(self)
} else {
Err(UnexpectedTokenError {
expected: expected.to_vec(),
found: Some(self),
found: Some(self.token),
})
.with_span(self.span)
}
}
}

8
src/lexer.rs
View file

@ -183,9 +183,17 @@ pub struct TokenStream<'source> {
}
impl<'source> TokenStream<'source> {
pub fn new(lexer: Lexer<'source, Token>) -> Self {
TokenStream { lexer }
}
pub fn source(&self) -> &'source str {
self.lexer.source()
}
pub fn span(&self) -> Span {
self.lexer.span()
}
}
impl<'source> Iterator for TokenStream<'source> {

89
src/parser.rs
View file

@ -1,6 +1,6 @@
use crate::error::UnexpectedTokenError;
use crate::error::{ExpectToken, InvalidArrayKeyError, ParseError, ResultExt, SpannedError};
use crate::lexer::Token;
use crate::lexer::{SpannedToken, Token, TokenStream};
use crate::num::parse_int;
use crate::string::parse_string;
use crate::{Key, Value};
@ -27,27 +27,24 @@ use std::num::ParseFloatError;
/// ```
///
pub fn parse(source: &str) -> Result<Value, SpannedError<ParseError>> {
Parser::new(source).parse()
Parser::new(source).parse_any()
}
pub struct Parser<'source> {
source: &'source str,
lexer: Lexer<'source, Token>,
tokens: TokenStream<'source>,
}
impl<'source> Parser<'source> {
pub fn new(source: &'source str) -> Self {
Parser {
source,
lexer: Token::lexer(source),
tokens: TokenStream::new(Token::lexer(source)),
}
}
pub fn parse(&mut self) -> Result<Value, SpannedError<ParseError>> {
let token = self
.lexer
.next()
.expect_token(&[
pub fn parse_any(&mut self) -> Result<Value, SpannedError<ParseError>> {
let token = self.tokens.next().expect_token(&[
Token::Bool,
Token::Integer,
Token::Float,
@ -55,25 +52,12 @@ impl<'source> Parser<'source> {
Token::Null,
Token::Array,
Token::SquareOpen,
])
.with_span(self.lexer.span())?;
let value = match token {
Token::Bool => Value::Bool(
self.lexer
.slice()
.to_ascii_lowercase()
.parse()
.with_span(self.lexer.span())?,
),
Token::Integer => {
Value::Int(parse_int(self.lexer.slice()).with_span(self.lexer.span())?)
}
Token::Float => {
Value::Float(parse_float(self.lexer.slice()).with_span(self.lexer.span())?)
}
Token::LiteralString => {
Value::String(parse_string(self.lexer.slice()).with_span(self.lexer.span())?)
}
])?;
let value = match token.token {
Token::Bool => Value::Bool(self.parse_bool(token)?),
Token::Integer => Value::Int(self.parse_int(token)?),
Token::Float => Value::Float(self.parse_float(token)?),
Token::LiteralString => Value::String(self.parse_string(token)?),
Token::Null => Value::Null,
Token::Array => Value::Array(self.parse_array(ArraySyntax::Long)?),
Token::SquareOpen => Value::Array(self.parse_array(ArraySyntax::Short)?),
@ -83,6 +67,26 @@ impl<'source> Parser<'source> {
Ok(value)
}
fn parse_bool(&self, token: SpannedToken) -> Result<bool, SpannedError<ParseError>> {
token
.slice()
.to_ascii_lowercase()
.parse()
.with_span(token.span)
}
fn parse_int(&self, token: SpannedToken) -> Result<i64, SpannedError<ParseError>> {
parse_int(token.slice()).with_span(token.span)
}
fn parse_float(&self, token: SpannedToken) -> Result<f64, SpannedError<ParseError>> {
parse_float(token.slice()).with_span(token.span)
}
fn parse_string(&self, token: SpannedToken) -> Result<String, SpannedError<ParseError>> {
parse_string(token.slice()).with_span(token.span)
}
fn parse_array(
&mut self,
syntax: ArraySyntax,
@ -90,14 +94,11 @@ impl<'source> Parser<'source> {
let mut builder = ArrayBuilder::default();
if syntax == ArraySyntax::Long {
self.lexer
.next()
.expect_token(&[Token::BracketOpen])
.with_span(self.lexer.span())?;
self.tokens.next().expect_token(&[Token::BracketOpen])?;
}
loop {
let key_or_value = match self.parse() {
let key_or_value = match self.parse_any() {
Ok(value) => value,
Err(err) => {
// trailing comma or empty array
@ -110,14 +111,14 @@ impl<'source> Parser<'source> {
}
}
};
let key_or_value_span = self.lexer.span();
let next = self
.lexer
.next()
.expect_token(&[syntax.close_bracket(), Token::Comma, Token::Arrow])
.with_span(self.lexer.span())?;
let key_or_value_span = self.tokens.span();
let next = self.tokens.next().expect_token(&[
syntax.close_bracket(),
Token::Comma,
Token::Arrow,
])?;
match next {
match next.token {
Token::BracketClose => {
builder.push_value(key_or_value);
break;
@ -130,7 +131,7 @@ impl<'source> Parser<'source> {
builder.push_value(key_or_value);
}
Token::Arrow => {
let value = self.parse()?;
let value = self.parse_any()?;
let key = match key_or_value {
Value::Int(int) => Key::Int(int),
Value::Float(float) => Key::Int(float as i64),
@ -144,10 +145,10 @@ impl<'source> Parser<'source> {
builder.push_key_value(key, value);
match self
.lexer
.tokens
.next()
.expect_token(&[syntax.close_bracket(), Token::Comma])
.with_span(self.lexer.span())?
.expect_token(&[syntax.close_bracket(), Token::Comma])?
.token
{
Token::BracketClose => {
break;