handle mallformed utf8 gameevent values

src/bin/all_test.rs

@@ -0,0 +1,67 @@
+use main_error::MainError;
+use rayon::prelude::*;
+use std::env;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+pub use tf_demo_parser::{Demo, DemoParser, Parse, ParseError, ParserState, Stream};
+
+#[cfg(feature = "jemallocator")]
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
+fn main() -> Result<(), MainError> {
+    #[cfg(feature = "better_panic")]
+    better_panic::install();
+
+    let args: Vec<_> = env::args().collect();
+    if args.len() < 2 {
+        println!("1 argument required");
+        return Ok(());
+    }
+    let path = args[1].clone();
+    let all = args
+        .get(2)
+        .map(|arg| arg.as_str() == "all")
+        .unwrap_or_default();
+
+    let files = gather_dir(path)?;
+    println!("found {} demo files", files.len());
+
+    let failures = files.par_iter().filter_map(|entry| {
+        let file = fs::read(&entry).unwrap();
+        let demo = Demo::new(file);
+        let parser = if all {
+            DemoParser::new_all(demo.get_stream())
+        } else {
+            DemoParser::new(demo.get_stream())
+        };
+        if let Err(e) = parser.parse() {
+            eprintln!("{}: {}", entry.to_str().unwrap(), e);
+            Some(entry)
+        } else {
+            None
+        }
+    });
+    println!("Found {} failures", failures.count());
+    Ok(())
+}
+
+fn gather_dir(path: impl AsRef<Path>) -> Result<Vec<PathBuf>, MainError> {
+    let mut files = Vec::with_capacity(512);
+
+    for res in fs::read_dir(path)? {
+        let entry = res?;
+
+        if entry.file_type()?.is_dir() {
+            files.extend_from_slice(&gather_dir(entry.path())?);
+        } else {
+            let entry_path = entry.path();
+            if entry_path.extension() == Some(OsStr::new("dem")) {
+                files.push(entry.path());
+            }
+        }
+    }
+
+    Ok(files)
+}

src/demo/message/gameevent.rs

@@ -9,12 +9,15 @@ use crate::demo::gamevent::{
     GameEvent, GameEventDefinition, GameEventEntry, GameEventValue, GameEventValueType,
     RawGameEvent,
 };
+use crate::demo::handle_utf8_error;
 use crate::demo::parser::ParseBitSkip;
 use crate::{GameEventError, Parse, ParseError, ParserState, ReadResult, Result, Stream};
 
 fn read_event_value(stream: &mut Stream, definition: &GameEventEntry) -> Result<GameEventValue> {
     Ok(match definition.kind {
-        GameEventValueType::String => GameEventValue::String(stream.read()?),
+        GameEventValueType::String => {
+            GameEventValue::String(stream.read().or_else(handle_utf8_error)?)
+        }
         GameEventValueType::Float => GameEventValue::Float(stream.read()?),
         GameEventValueType::Long => GameEventValue::Long(stream.read()?),
         GameEventValueType::Short => GameEventValue::Short(stream.read()?),

src/demo/message/usermessage.rs

@@ -3,6 +3,7 @@ use num_enum::TryFromPrimitive;
 use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 
+use crate::demo::handle_utf8_error;
 use crate::demo::parser::ParseBitSkip;
 use crate::{ParseError, ReadResult, Result, Stream};
 
@@ -145,13 +146,6 @@ pub struct SayText2Message {
     pub text: String,
 }
 
-fn handle_utf8_error(error: ReadError) -> ReadResult<String> {
-    match error {
-        ReadError::Utf8Error(_) => Ok("-- Malformed utf8 --".into()),
-        _ => Err(error),
-    }
-}
-
 impl BitRead<LittleEndian> for SayText2Message {
     fn read(stream: &mut Stream) -> ReadResult<Self> {
         let client = stream.read()?;

src/demo/mod.rs

@@ -1,4 +1,5 @@
-use bitstream_reader::{BitBuffer, BitStream, LittleEndian};
+use crate::ReadResult;
+use bitstream_reader::{BitBuffer, BitStream, LittleEndian, ReadError};
 
 pub mod gameevent_gen;
 pub mod gamevent;
@@ -28,3 +29,10 @@ impl Demo {
         self.stream.clone()
     }
 }
+
+pub(crate) fn handle_utf8_error(error: ReadError) -> ReadResult<String> {
+    match error {
+        ReadError::Utf8Error(_) => Ok("-- Malformed utf8 --".into()),
+        _ => Err(error),
+    }
+}