robin/vdf-reader
1
0
Fork 0
mirror of https://codeberg.org/icewind/vdf-reader.git synced 2026-06-03 18:14:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

unquote strings

Browse source
This commit is contained in:
Robin Appelman 2023-12-15 16:04:31 +01:00
commit c7d87804f8
4 changed files with 96 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/entry/table.rs
View file

@ -31,8 +31,8 @@ impl Table {
pub fn load(reader: &mut Reader) -> Result<Table> { pub fn load(reader: &mut Reader) -> Result<Table> {
let mut map = HashMap::new(); let mut map = HashMap::new();
loop { while let Some(event) = reader.event() {
match reader.event()? { match event? {
Event::Entry { Event::Entry {
key: Item::Statement { .. }, key: Item::Statement { .. },
span, span,
@ -58,7 +58,7 @@ impl Table {
insert(&mut map, name.into(), Table::load(reader)?.into()) insert(&mut map, name.into(), Table::load(reader)?.into())
} }
Event::GroupEnd { .. } | Event::End { .. } => break, Event::GroupEnd { .. } => break,
} }
} }

36
src/parser.rs
View file

@ -16,11 +16,19 @@ pub enum Token {
#[display("end of group")] #[display("end of group")]
GroupEnd, GroupEnd,
/// An enclosed or bare item. /// An enclosed or bare item.
#[regex("(\"([^\"\\\\]|\\\\.)*\")|([^# \t\n{}\"][^ \"\t\n]*)", priority = 0)] #[regex("[^# \t\n{}\"][^ \"\t\n]*", priority = 0)]
#[display("item")] #[display("item")]
Item, Item,
/// An enclosed or bare item.
#[regex("\"([^\"\\\\]|\\\\.)*\"")]
#[display("item")]
QuotedItem,
/// An enclosed or bare statement. /// An enclosed or bare statement.
#[regex("(\"#([^\"\\\\]|\\\\.)*\")|(#[^ \"\t\n]+)")] #[regex("\"#([^\"\\\\]|\\\\.)*\"")]
#[display("statement")]
QuotedStatement,
/// An enclosed or bare statement.
#[regex("#[^ \"\t\n]+")]
#[display("statement")] #[display("statement")]
Statement, Statement,
} }
@ -46,11 +54,11 @@ mod tests {
#[test] #[test]
fn next() { fn next() {
assert_eq!(get_token("test"), Some(Ok(Token::Item))); assert_eq!(get_token("test"), Some(Ok(Token::Item)));
assert_eq!(get_token("\"test\""), Some(Ok(Token::Item))); assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("\"\""), Some(Ok(Token::Item))); assert_eq!(get_token("\"\""), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("\"\" "), Some(Ok(Token::Item))); assert_eq!(get_token("\"\" "), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("#test"), Some(Ok(Token::Statement))); assert_eq!(get_token("#test"), Some(Ok(Token::Statement)));
assert_eq!(get_token("\"#test\""), Some(Ok(Token::Statement))); assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
assert_eq!(get_token("{"), Some(Ok(Token::GroupStart))); assert_eq!(get_token("{"), Some(Ok(Token::GroupStart)));
assert_eq!(get_token("}"), Some(Ok(Token::GroupEnd))); assert_eq!(get_token("}"), Some(Ok(Token::GroupEnd)));
assert_eq!(get_token("//test more"), None); assert_eq!(get_token("//test more"), None);
@ -67,12 +75,12 @@ mod tests {
assert_eq!(get_token("lol}"), Some(Ok(Token::Item))); assert_eq!(get_token("lol}"), Some(Ok(Token::Item)));
assert_eq!(get_token("#lol}"), Some(Ok(Token::Statement))); assert_eq!(get_token("#lol}"), Some(Ok(Token::Statement)));
assert_eq!(get_token("\"test\""), Some(Ok(Token::Item))); assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("\"#test\""), Some(Ok(Token::Statement))); assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
assert_eq!(get_token("\"te\\\"st\""), Some(Ok(Token::Item))); assert_eq!(get_token("\"te\\\"st\""), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("\"te\\st\""), Some(Ok(Token::Item))); assert_eq!(get_token("\"te\\st\""), Some(Ok(Token::QuotedItem)));
assert_eq!(get_token("\"#te\\\"st\""), Some(Ok(Token::Statement))); assert_eq!(get_token("\"#te\\\"st\""), Some(Ok(Token::QuotedStatement)));
} }
#[test] #[test]
@ -89,12 +97,12 @@ mod tests {
Ok(vec![ Ok(vec![
(Token::Item, "foo"), (Token::Item, "foo"),
(Token::GroupStart, "{"), (Token::GroupStart, "{"),
(Token::Item, r#""asd""#), (Token::QuotedItem, r#""asd""#),
(Token::Item, r#""bar""#), (Token::QuotedItem, r#""bar""#),
(Token::Statement, r#"#include"#), (Token::Statement, r#"#include"#),
(Token::Item, r#"other"#), (Token::Item, r#"other"#),
(Token::Item, r#"empty"#), (Token::Item, r#"empty"#),
(Token::Item, r#""""#), (Token::QuotedItem, r#""""#),
(Token::GroupEnd, "}") (Token::GroupEnd, "}")
]) ])
) )

107
src/reader.rs
View file

@ -44,9 +44,6 @@ pub enum Event<'a> {
value: Item<'a>, value: Item<'a>,
span: Span, span: Span,
}, },
/// EOF has been reached.
End { span: Span },
} }
impl Event<'_> { impl Event<'_> {
@ -56,7 +53,6 @@ impl Event<'_> {
Event::GroupStart { span, .. } => span.clone(), Event::GroupStart { span, .. } => span.clone(),
Event::GroupEnd { span, .. } => span.clone(), Event::GroupEnd { span, .. } => span.clone(),
Event::Entry { span, .. } => span.clone(), Event::Entry { span, .. } => span.clone(),
Event::End { span, .. } => span.clone(),
} }
} }
} }
@ -79,96 +75,119 @@ impl<'a> From<&'a str> for Reader<'a> {
impl<'a> Reader<'a> { impl<'a> Reader<'a> {
/// Get the next event, this does copies. /// Get the next event, this does copies.
#[allow(dead_code)] #[allow(dead_code)]
pub fn event(&mut self) -> Result<Event> { pub fn event(&mut self) -> Option<Result<Event>> {
const VALID_KEY: &[Token] = &[
Token::Item,
Token::QuotedItem,
Token::GroupEnd,
Token::Statement,
Token::QuotedStatement,
];
let key = match self.lexer.next() { let key = match self.lexer.next() {
None => { None => {
return Ok(Event::End { return None;
span: self.lexer.span(),
})
} }
Some((Err(_), span)) => { Some((Err(_), span)) => {
return Err(NoValidTokenError::new( return Some(Err(NoValidTokenError::new(
&[Token::Item, Token::GroupEnd, Token::Statement], VALID_KEY,
span.into(), span.into(),
self.content.into(), self.content.into(),
) )
.into()); .into()));
}
Some((Ok(Token::GroupEnd), span)) => return Ok(Event::GroupEnd { span }),
Some((Ok(Token::GroupStart), span)) => {
return Err(UnexpectedTokenError::new(
&[Token::Item, Token::GroupEnd, Token::Statement],
Some(Token::GroupStart),
span.into(),
self.content.into(),
)
.into())
} }
Some((Ok(Token::GroupEnd), span)) => return Some(Ok(Event::GroupEnd { span })),
Some((Ok(Token::Item), span)) => Item::Value { Some((Ok(Token::Item), span)) => Item::Value {
content: string(self.lexer.slice()), content: string(self.lexer.slice()),
span, span,
}, },
Some((Ok(Token::QuotedItem), span)) => Item::Value {
content: quoted_string(self.lexer.slice()),
span,
},
Some((Ok(Token::Statement), span)) => Item::Statement { Some((Ok(Token::Statement), span)) => Item::Statement {
content: string(self.lexer.slice()), content: string(self.lexer.slice()),
span, span,
}, },
Some((Ok(Token::QuotedStatement), span)) => Item::Statement {
content: quoted_string(self.lexer.slice()),
span,
},
Some((Ok(token), span)) => {
return Some(Err(UnexpectedTokenError::new(
VALID_KEY,
Some(token),
span.into(),
self.content.into(),
)
.into()))
}
}; };
const VALID_VALUE: &[Token] = &[Token::Item, Token::QuotedItem, Token::GroupStart];
let value = match self.lexer.next() { let value = match self.lexer.next() {
None => { None => {
return Err(UnexpectedTokenError::new( return Some(Err(UnexpectedTokenError::new(
&[Token::Item, Token::GroupEnd, Token::Statement], VALID_VALUE,
None, None,
self.lexer.span().into(), self.lexer.span().into(),
self.content.into(), self.content.into(),
) )
.into()); .into()));
} }
Some((Err(_), span)) => { Some((Err(_), span)) => {
return Err(NoValidTokenError::new( return Some(Err(NoValidTokenError::new(
&[Token::Item, Token::GroupEnd, Token::Statement], VALID_VALUE,
span.into(), span.into(),
self.content.into(), self.content.into(),
) )
.into()); .into()));
}
Some((Ok(Token::GroupEnd), span)) => {
return Err(UnexpectedTokenError::new(
&[Token::Item, Token::GroupStart, Token::Statement],
Some(Token::GroupEnd),
span.into(),
self.content.into(),
)
.into())
} }
Some((Ok(Token::GroupStart), span)) => { Some((Ok(Token::GroupStart), span)) => {
return Ok(Event::GroupStart { return Some(Ok(Event::GroupStart {
name: key.into_content(), name: key.into_content(),
span, span,
}) }))
} }
Some((Ok(Token::QuotedItem), span)) => Item::Value {
content: quoted_string(self.lexer.slice()),
span,
},
Some((Ok(Token::Item), span)) => Item::Value { Some((Ok(Token::Item), span)) => Item::Value {
content: string(self.lexer.slice()), content: string(self.lexer.slice()),
span, span,
}, },
Some((Ok(Token::Statement), span)) => Item::Statement { Some((Ok(token), span)) => {
content: string(self.lexer.slice()), return Some(Err(UnexpectedTokenError::new(
span, VALID_VALUE,
}, Some(token),
span.into(),
self.content.into(),
)
.into()))
}
}; };
let span = key.span().start..value.span().end; let span = key.span().start..value.span().end;
Ok(Event::Entry { key, value, span }) Some(Ok(Event::Entry { key, value, span }))
} }
} }
fn quoted_string(source: &str) -> Cow<str> {
string(&source[1..source.len() - 1])
}
fn string(source: &str) -> Cow<str> { fn string(source: &str) -> Cow<str> {
if source.contains(r#"\""#) || source.contains(r#"\\"#) { if source.contains(r#"\""#) || source.contains(r#"\\"#) {
let mut buffer = source.bytes(); let mut buffer = source.bytes();

18
tests/snapshots/parse__parse.snap
View file

@ -3,14 +3,14 @@ source: tests/parse.rs
expression: parsed expression: parsed
--- ---
Table({ Table({
"\"LightmappedGeneric\"": Table(Table({ "LightmappedGeneric": Table(Table({
"\"$detailblendfactor\"": Value(Value("\"1\"")), "$detailblendmode": Value(Value("0")),
"\"%keywords\"": Value(Value("\"tf\"")), "$detailscale": Value(Value("1.9")),
"\"$ssbump\"": Value(Value("\"1\"")), "%keywords": Value(Value("tf")),
"\"$baseTexture\"": Value(Value("\"cp_mountainlab/concrete/concretefloor003\"")), "$bumpmap": Value(Value("concrete/concretefloor007b_height-ssbump")),
"\"$detailblendmode\"": Value(Value("\"0\"")), "$detail": Value(Value("overlays/detail001")),
"\"$detailscale\"": Value(Value("\"1.9\"")), "$baseTexture": Value(Value("cp_mountainlab/concrete/concretefloor003")),
"\"$bumpmap\"": Value(Value("\"concrete/concretefloor007b_height-ssbump\"")), "$detailblendfactor": Value(Value("1")),
"\"$detail\"": Value(Value("\"overlays/detail001\"")), "$ssbump": Value(Value("1")),
})), })),
}) })