better mallformed utf8 handling

src/demo/data/mod.rs

@@ -1,3 +1,89 @@
 pub mod userinfo;
 
+use bitbuffer::{BitRead, BitReadStream, BitWrite, BitWriteStream, Endianness};
+use serde::{Deserialize, Serialize};
+use std::fmt::{Debug, Display, Formatter};
+
 pub use userinfo::UserInfo;
+
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
+#[derive(Eq, PartialEq, Serialize, Deserialize, Clone)]
+pub enum MaybeUtf8String {
+    Valid(String),
+    Invalid(Vec<u8>),
+}
+
+impl Default for MaybeUtf8String {
+    fn default() -> Self {
+        MaybeUtf8String::Valid(String::new())
+    }
+}
+
+impl AsRef<str> for MaybeUtf8String {
+    fn as_ref(&self) -> &str {
+        match self {
+            MaybeUtf8String::Valid(s) => s.as_str(),
+            MaybeUtf8String::Invalid(_) => "-- Malformed utf8 --",
+        }
+    }
+}
+
+impl Debug for MaybeUtf8String {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            MaybeUtf8String::Valid(s) => Debug::fmt(s, f),
+            MaybeUtf8String::Invalid(b) => f
+                .debug_struct("MaybeUtf8String::Invalid")
+                .field("data", b)
+                .finish(),
+        }
+    }
+}
+
+impl Display for MaybeUtf8String {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            MaybeUtf8String::Valid(s) => Display::fmt(s, f),
+            MaybeUtf8String::Invalid(_) => write!(f, "-- Malformed utf8 --"),
+        }
+    }
+}
+
+impl MaybeUtf8String {
+    pub fn as_bytes(&self) -> &[u8] {
+        match self {
+            MaybeUtf8String::Valid(s) => s.as_bytes(),
+            MaybeUtf8String::Invalid(b) => b.as_ref(),
+        }
+    }
+}
+
+impl<'a, E: Endianness> BitRead<'a, E> for MaybeUtf8String {
+    fn read(stream: &mut BitReadStream<'a, E>) -> bitbuffer::Result<Self> {
+        match String::read(stream) {
+            Ok(str) => Ok(MaybeUtf8String::Valid(str)),
+            Err(bitbuffer::BitError::Utf8Error(_, size)) => {
+                stream.set_pos(stream.pos() - size * 8)?;
+                let data = stream.read_sized(size)?;
+                Ok(MaybeUtf8String::Invalid(data))
+            }
+            Err(e) => Err(e),
+        }
+    }
+}
+
+impl<E: Endianness> BitWrite<E> for MaybeUtf8String {
+    fn write(&self, stream: &mut BitWriteStream<E>) -> bitbuffer::Result<()> {
+        stream.write_bytes(self.as_bytes())?;
+        stream.write(&0u8)
+    }
+}
+
+impl Into<String> for MaybeUtf8String {
+    fn into(self) -> String {
+        match self {
+            MaybeUtf8String::Valid(s) => s,
+            MaybeUtf8String::Invalid(_) => "-- Malformed utf8 --".into(),
+        }
+    }
+}

src/demo/gamevent.rs

@@ -1,5 +1,5 @@
 pub use super::gameevent_gen::{GameEvent, GameEventType};
-use crate::demo::handle_utf8_error;
+use crate::demo::data::MaybeUtf8String;
 use crate::demo::message::gameevent::GameEventTypeId;
 use crate::{GameEventError, Result, Stream};
 use bitbuffer::{BitRead, BitWrite, BitWriteStream, LittleEndian};
@@ -59,7 +59,7 @@ pub enum GameEventValueType {
 #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum GameEventValue {
-    String(String),
+    String(MaybeUtf8String),
     Float(f32),
     Long(u32),
     Short(u16),
@@ -70,9 +70,7 @@ pub enum GameEventValue {
 
 fn read_event_value(stream: &mut Stream, definition: &GameEventEntry) -> Result<GameEventValue> {
     Ok(match definition.kind {
-        GameEventValueType::String => {
-            GameEventValue::String(stream.read().or_else(handle_utf8_error)?)
-        }
+        GameEventValueType::String => GameEventValue::String(stream.read()?),
         GameEventValueType::Float => GameEventValue::Float(stream.read()?),
         GameEventValueType::Long => GameEventValue::Long(stream.read()?),
         GameEventValueType::Short => GameEventValue::Short(stream.read()?),
@@ -121,6 +119,12 @@ impl EventValue for String {
     }
 }
 
+impl EventValue for MaybeUtf8String {
+    fn value_type() -> GameEventValueType {
+        GameEventValueType::String
+    }
+}
+
 impl EventValue for f32 {
     fn value_type() -> GameEventValueType {
         GameEventValueType::Float

src/demo/message/generated.rs

@@ -1,5 +1,6 @@
-use crate::Stream;
 /// Messages that consists only of primitives and string and can be derived
+use crate::demo::data::MaybeUtf8String;
+use crate::Stream;
 use bitbuffer::{BitRead, BitWrite, LittleEndian};
 use serde::{Deserialize, Serialize};
 
@@ -49,7 +50,7 @@ pub struct SignOnStateMessage {
 #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
 #[derive(BitRead, BitWrite, Debug, PartialEq, Serialize, Deserialize, Clone)]
 pub struct PrintMessage {
-    pub value: String,
+    pub value: MaybeUtf8String,
 }
 
 #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]

src/demo/message/usermessage.rs

@@ -1,8 +1,7 @@
 use bitbuffer::{BitError, BitRead, BitWrite, BitWriteStream, LittleEndian};
 use serde::{Deserialize, Serialize};
 
-use crate::demo::handle_utf8_error;
-
+use crate::demo::data::MaybeUtf8String;
 use crate::demo::parser::analyser::UserId;
 use crate::{ReadResult, Stream};
 
@@ -228,14 +227,14 @@ pub struct SayText2Message {
     pub client: UserId,
     pub raw: u8,
     pub kind: ChatMessageKind,
-    pub from: Option<String>,
-    pub text: String,
+    pub from: Option<MaybeUtf8String>,
+    pub text: MaybeUtf8String,
 }
 
 impl SayText2Message {
     pub fn plain_text(&self) -> String {
         // 1: normal, 2: old colors, 3: team, 4: location, 5 achievement, 6 custom
-        let mut text = self.text.replace(|c| c <= char::from(6), "");
+        let mut text = self.text.to_string().replace(|c| c <= char::from(6), "");
         // 7: 6-char hex
         while let Some(pos) = text.chars().enumerate().find_map(|(index, c)| {
             if c == char::from(7) {
@@ -272,18 +271,18 @@ impl BitRead<'_, LittleEndian> for SayText2Message {
     fn read(stream: &mut Stream) -> ReadResult<Self> {
         let client = UserId(stream.read()?);
         let raw = stream.read()?;
-        let (kind, from, text): (ChatMessageKind, Option<String>, String) =
+        let (kind, from, text): (ChatMessageKind, Option<MaybeUtf8String>, MaybeUtf8String) =
             if stream.read::<u8>()? == 1 {
                 stream.set_pos(stream.pos() - 8)?;
 
-                let text: String = stream.read().or_else(handle_utf8_error)?;
+                let text: MaybeUtf8String = stream.read()?;
                 (ChatMessageKind::ChatAll, None, text)
             } else {
                 stream.set_pos(stream.pos() - 8)?;
 
                 let kind = stream.read()?;
-                let from = stream.read().or_else(handle_utf8_error)?;
-                let text = stream.read().or_else(handle_utf8_error)?;
+                let from = stream.read()?;
+                let text = stream.read()?;
 
                 // ends with 2 0 bytes?
                 if stream.bits_left() >= 16 {
@@ -307,7 +306,7 @@ impl BitWrite<LittleEndian> for SayText2Message {
         u8::from(self.client).write(stream)?;
         self.raw.write(stream)?;
 
-        if let Some(from) = self.from.as_deref() {
+        if let Some(from) = self.from.as_ref().map(|s| s.as_ref()) {
             self.kind.write(stream)?;
             from.write(stream)?;
             self.text.write(stream)?;

src/demo/mod.rs

@@ -1,5 +1,4 @@
-use crate::ReadResult;
-use bitbuffer::{BitError, BitReadBuffer, BitReadStream, LittleEndian};
+use bitbuffer::{BitReadBuffer, BitReadStream, LittleEndian};
 
 pub mod data;
 pub mod gameevent_gen;
@@ -40,10 +39,3 @@ impl Demo<'static> {
         Demo { stream }
     }
 }
-
-pub(crate) fn handle_utf8_error(error: BitError) -> ReadResult<String> {
-    match error {
-        BitError::Utf8Error(_, _) => Ok("-- Malformed utf8 --".into()),
-        _ => Err(error),
-    }
-}

src/demo/parser/analyser.rs

@@ -29,7 +29,11 @@ impl ChatMessage {
     pub fn from_message(message: &SayText2Message, tick: u32) -> Self {
         ChatMessage {
             kind: message.kind,
-            from: message.from.clone().unwrap_or_default(),
+            from: message
+                .from
+                .as_ref()
+                .map(|s| s.to_string())
+                .unwrap_or_default(),
             text: message.plain_text(),
             tick,
         }
@@ -328,7 +332,7 @@ impl Death {
             assister,
             tick,
             killer: UserId::from(event.attacker),
-            weapon: event.weapon.clone(),
+            weapon: event.weapon.to_string(),
             victim: UserId::from(event.user_id),
         }
     }
@@ -416,7 +420,7 @@ impl Analyser {
         if let UserMessage::SayText2(text_message) = message {
             if text_message.kind == ChatMessageKind::NameChange {
                 if let Some(from) = text_message.from.clone() {
-                    self.change_name(from, text_message.text.clone());
+                    self.change_name(from.into(), text_message.plain_text());
                 }
             } else {
                 self.state