mirror of
https://codeberg.org/demostf/parser.git
synced 2026-06-03 18:24:05 +02:00
handle mallformed utf8 gameevent values
This commit is contained in:
parent
d952c5c4bc
commit
6599a5d6d2
8 changed files with 201 additions and 9 deletions
95
Cargo.lock
generated
95
Cargo.lock
generated
|
|
@ -135,6 +135,47 @@ dependencies = [
|
|||
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "0.1.12"
|
||||
|
|
@ -159,6 +200,11 @@ name = "difference"
|
|||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
|
|
@ -260,6 +306,14 @@ name = "memchr"
|
|||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "no-panic"
|
||||
version = "0.1.12"
|
||||
|
|
@ -278,6 +332,15 @@ dependencies = [
|
|||
"autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.4.2"
|
||||
|
|
@ -412,6 +475,28 @@ dependencies = [
|
|||
"proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.56"
|
||||
|
|
@ -585,6 +670,7 @@ dependencies = [
|
|||
"num_enum 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parse-display 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_repr 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -656,9 +742,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum clicolors-control 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e"
|
||||
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
"checksum console 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8ca57c2c14b8a2bf3105bc9d15574aad80babf6a9c44b1058034cdf8bd169628"
|
||||
"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
|
||||
"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
|
||||
"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
|
||||
"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
|
||||
"checksum ctor 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc"
|
||||
"checksum derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "942ca430eef7a3806595a6737bc388bf51adb888d3fc0dd1b50f1c170167ee3a"
|
||||
"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
|
||||
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
|
||||
"checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
"checksum enumflags2 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "33121c8782ba948ba332dab29311b026a8716dc65a1599e5b88f392d38496af8"
|
||||
"checksum enumflags2_derive 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894"
|
||||
|
|
@ -673,8 +764,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b"
|
||||
"checksum main_error 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3516df0fb44d98fe6d6e859d224adfb7b6686447937e5b96308d6061595eed04"
|
||||
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
|
||||
"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
|
||||
"checksum no-panic 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a11c6db47b62f887ae15a0d02c5b24eb9a815536812bc46ca45305a6d22e5675"
|
||||
"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
|
||||
"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
|
||||
"checksum num_enum 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "be601e38e20a6f3d01049d85801cb9b7a34a8da7a0da70df507bbde7735058c8"
|
||||
"checksum num_enum_derive 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b59f30f6a043f2606adbd0addbf1eef6f2e28e8c4968918b63b7ff97ac0db2a7"
|
||||
"checksum output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
|
||||
|
|
@ -689,6 +782,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0319972dcae462681daf4da1adeeaa066e3ebd29c69be96c6abb1259d2ee2bcc"
|
||||
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
|
||||
"checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098"
|
||||
"checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9"
|
||||
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
|
||||
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ path = "src/lib.rs"
|
|||
name = "parse_demo"
|
||||
path = "src/bin/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "all_test"
|
||||
path = "src/bin/all_test.rs"
|
||||
|
||||
[dependencies]
|
||||
bitstream_reader = { version = "0.7", path = "../../bitbuffer" }
|
||||
num_enum = "0.4"
|
||||
|
|
@ -27,6 +31,7 @@ main_error = "0.1.0"
|
|||
jemallocator = { version = "0.3", optional = true }
|
||||
better-panic = { version = "0.1", optional = true }
|
||||
no-panic = { version = "0.1", optional = true }
|
||||
rayon = "1.3.0"
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = "0.6"
|
||||
|
|
|
|||
|
|
@ -22,3 +22,10 @@ members = ["."]
|
|||
[[bin]]
|
||||
name = "fuzz_target_1"
|
||||
path = "fuzz_targets/fuzz_target_1.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "real_inputs"
|
||||
path = "fuzz_targets/real_inputs.rs"
|
||||
|
||||
[profile.dev]
|
||||
opt-level = 2
|
||||
13
fuzz/fuzz_targets/real_inputs.rs
Normal file
13
fuzz/fuzz_targets/real_inputs.rs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#![no_main]
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
|
||||
pub use tf_demo_parser::{Demo, DemoParser, Parse, ParseError, ParserState, Stream};
|
||||
|
||||
fn fuzz(data: &[u8]) {
|
||||
let data = data.to_vec();
|
||||
let demo = Demo::new(data);
|
||||
let parser = DemoParser::new_all(demo.get_stream());
|
||||
let _ = parser.parse();
|
||||
}
|
||||
|
||||
fuzz_target!(|data: &[u8]| { fuzz(data) });
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
use main_error::MainError;
|
||||
use rayon::prelude::*;
|
||||
use std::env;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
pub use tf_demo_parser::{Demo, DemoParser, Parse, ParseError, ParserState, Stream};
|
||||
|
||||
#[cfg(feature = "jemallocator")]
|
||||
#[global_allocator]
|
||||
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||
|
||||
fn main() -> Result<(), MainError> {
|
||||
#[cfg(feature = "better_panic")]
|
||||
better_panic::install();
|
||||
|
||||
let args: Vec<_> = env::args().collect();
|
||||
if args.len() < 2 {
|
||||
println!("1 argument required");
|
||||
return Ok(());
|
||||
}
|
||||
let path = args[1].clone();
|
||||
let all = args
|
||||
.get(2)
|
||||
.map(|arg| arg.as_str() == "all")
|
||||
.unwrap_or_default();
|
||||
|
||||
let files = gather_dir(path)?;
|
||||
println!("found {} demo files", files.len());
|
||||
|
||||
let failures = files.par_iter().filter_map(|entry| {
|
||||
let file = fs::read(&entry).unwrap();
|
||||
let demo = Demo::new(file);
|
||||
let parser = if all {
|
||||
DemoParser::new_all(demo.get_stream())
|
||||
} else {
|
||||
DemoParser::new(demo.get_stream())
|
||||
};
|
||||
if let Err(e) = parser.parse() {
|
||||
eprintln!("{}: {}", entry.to_str().unwrap(), e);
|
||||
Some(entry)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
println!("Found {} failures", failures.count());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gather_dir(path: impl AsRef<Path>) -> Result<Vec<PathBuf>, MainError> {
|
||||
let mut files = Vec::with_capacity(512);
|
||||
|
||||
for res in fs::read_dir(path)? {
|
||||
let entry = res?;
|
||||
|
||||
if entry.file_type()?.is_dir() {
|
||||
files.extend_from_slice(&gather_dir(entry.path())?);
|
||||
} else {
|
||||
let entry_path = entry.path();
|
||||
if entry_path.extension() == Some(OsStr::new("dem")) {
|
||||
files.push(entry.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
|
@ -9,12 +9,15 @@ use crate::demo::gamevent::{
|
|||
GameEvent, GameEventDefinition, GameEventEntry, GameEventValue, GameEventValueType,
|
||||
RawGameEvent,
|
||||
};
|
||||
use crate::demo::handle_utf8_error;
|
||||
use crate::demo::parser::ParseBitSkip;
|
||||
use crate::{GameEventError, Parse, ParseError, ParserState, ReadResult, Result, Stream};
|
||||
|
||||
fn read_event_value(stream: &mut Stream, definition: &GameEventEntry) -> Result<GameEventValue> {
|
||||
Ok(match definition.kind {
|
||||
GameEventValueType::String => GameEventValue::String(stream.read()?),
|
||||
GameEventValueType::String => {
|
||||
GameEventValue::String(stream.read().or_else(handle_utf8_error)?)
|
||||
}
|
||||
GameEventValueType::Float => GameEventValue::Float(stream.read()?),
|
||||
GameEventValueType::Long => GameEventValue::Long(stream.read()?),
|
||||
GameEventValueType::Short => GameEventValue::Short(stream.read()?),
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use num_enum::TryFromPrimitive;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use crate::demo::handle_utf8_error;
|
||||
use crate::demo::parser::ParseBitSkip;
|
||||
use crate::{ParseError, ReadResult, Result, Stream};
|
||||
|
||||
|
|
@ -145,13 +146,6 @@ pub struct SayText2Message {
|
|||
pub text: String,
|
||||
}
|
||||
|
||||
fn handle_utf8_error(error: ReadError) -> ReadResult<String> {
|
||||
match error {
|
||||
ReadError::Utf8Error(_) => Ok("-- Malformed utf8 --".into()),
|
||||
_ => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
impl BitRead<LittleEndian> for SayText2Message {
|
||||
fn read(stream: &mut Stream) -> ReadResult<Self> {
|
||||
let client = stream.read()?;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use bitstream_reader::{BitBuffer, BitStream, LittleEndian};
|
||||
use crate::ReadResult;
|
||||
use bitstream_reader::{BitBuffer, BitStream, LittleEndian, ReadError};
|
||||
|
||||
pub mod gameevent_gen;
|
||||
pub mod gamevent;
|
||||
|
|
@ -28,3 +29,10 @@ impl Demo {
|
|||
self.stream.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn handle_utf8_error(error: ReadError) -> ReadResult<String> {
|
||||
match error {
|
||||
ReadError::Utf8Error(_) => Ok("-- Malformed utf8 --".into()),
|
||||
_ => Err(error),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue