entity fixes and test

Cargo.lock

@@ -67,16 +67,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "bitstream_reader"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bitstream_reader_derive 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitstream_reader_derive 0.6.0",
  "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "bitstream_reader_derive"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
  "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -493,7 +491,7 @@ name = "tf-demo-parser"
 version = "0.1.0"
 dependencies = [
  "better-panic 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "bitstream_reader 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitstream_reader 0.6.0",
  "enum-primitive-derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "enumflags2 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "enumflags2_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -558,8 +556,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum backtrace-sys 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "82a830b4ef2d1124a711c71d263c5abdc710ef8e907bd508c88be475cebc422b"
 "checksum better-panic 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "64714970eb2081691f85a923b78a490fd7c3e256dcc83c7a5177314586563356"
 "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
-"checksum bitstream_reader 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91461c33313c2f8b753c88f40a2ba76ef3bf02d9b411e39acaf88263947b778c"
-"checksum bitstream_reader_derive 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "14e4573210a0cc968772847e1db9705e546f60cc1525c5cbfbf470dfa1fa391c"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "ce400c638d48ee0e9ab75aef7997609ec57367ccfe1463f21bf53c3eca67bf46"
 "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"

Cargo.toml

@@ -13,7 +13,7 @@ name = "parse_demo"
 path = "src/bin/main.rs"
 
 [dependencies]
-bitstream_reader = "0.6"
+bitstream_reader = { version = "0.6", path = "../../bitbuffer" }
 enum-primitive-derive = "0.1"
 num-traits = "0.2"
 enumflags2 = "0.5"

src/demo/message/packetentities.rs

@@ -1,5 +1,6 @@
 use bitstream_reader::{BitRead, BitReadSized, LittleEndian};
 use serde::{Deserialize, Serialize};
+use serde_repr::{Deserialize_repr, Serialize_repr};
 
 use crate::demo::message::stringtable::{log_base2, read_var_int};
 use crate::demo::packet::datatable::{SendTable, SendTableName, ServerClass};
@@ -31,8 +32,9 @@ impl FromStr for EntityId {
     }
 }
 
-#[derive(BitRead, Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(BitRead, Clone, Copy, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
 #[discriminant_bits = 2]
+#[repr(u8)]
 pub enum PVS {
     Preserve = 0,
     Leave = 1,

src/demo/packet/datatable.rs

@@ -30,10 +30,19 @@ impl From<u16> for ClassId {
     }
 }
 
+#[derive(BitRead, PartialEq, Eq, Hash, Debug, Serialize, Deserialize, Clone, Display)]
+pub struct ServerClassName(Rc<String>);
+
+impl From<String> for ServerClassName {
+    fn from(value: String) -> Self {
+        Self(Rc::new(value))
+    }
+}
+
 #[derive(BitRead, Debug, Clone)]
 pub struct ServerClass {
     pub id: ClassId,
-    pub name: String,
+    pub name: ServerClassName,
     pub data_table: SendTableName,
 }
 

src/demo/parser/handler.rs

@@ -54,6 +54,7 @@ impl<T: MessageHandler> DemoHandler<T> {
                 }
             }
             Packet::Message(packet) | Packet::Sigon(packet) => {
+                //self.tick = packet.tick;
                 for message in packet.messages {
                     match message {
                         Message::NetTick(message) => self.tick = message.tick,
@@ -108,12 +109,13 @@ impl<T: MessageHandler> DemoHandler<T> {
     fn handle_message(&mut self, message: Message) {
         let message_type = message.get_message_type();
         if ParserState::does_handle(message_type) {
-            self.state_handler.handle_message(message, self.tick);
+            if let Some(message) = self.state_handler.handle_message(message, self.tick) {
-            return;
+                if T::does_handle(message_type) {
-        }
+                    self.analyser.handle_message(message, self.tick);
-        if Analyser::does_handle(message_type) {
+                }
+            }
+        } else if T::does_handle(message_type) {
             self.analyser.handle_message(message, self.tick);
-            return;
         }
     }
 

src/demo/parser/state.rs

@@ -107,14 +107,11 @@ impl ParserState {
             Self::does_handle(message_type) || (self.analyser_handles)(message_type)
         }
     }
-}
 
-impl MessageHandler for ParserState {
+    pub fn does_handle(message_type: MessageType) -> bool {
-    type Output = Self;
-
-    fn does_handle(message_type: MessageType) -> bool {
         match message_type {
             MessageType::ServerInfo
+            | MessageType::NetTick
             | MessageType::GameEventList
             | MessageType::CreateStringTable
             | MessageType::PacketEntities
@@ -123,46 +120,49 @@ impl MessageHandler for ParserState {
         }
     }
 
-    fn handle_message(&mut self, message: Message, _tick: u32) {
+    pub fn handle_message(&mut self, message: Message, _tick: u32) -> Option<Message> {
         match message {
             Message::ServerInfo(message) => {
                 self.demo_meta.version = message.version;
                 self.demo_meta.game = message.game;
-                self.demo_meta.interval_per_tick = message.interval_per_tick
+                self.demo_meta.interval_per_tick = message.interval_per_tick;
+                None
             }
             Message::GameEventList(message) => {
                 self.event_definitions = message.event_list;
+                None
             }
-            Message::PacketEntities(message) => {
+            Message::PacketEntities(ent_message) => {
-                if message.updated_base_line {
+                if ent_message.updated_base_line {
-                    let old_index = message.base_line as usize;
+                    let old_index = ent_message.base_line as usize;
                     let new_index = 1 - old_index;
                     self.instance_baselines.swap(0, 1);
                     //self.instance_baselines[new_index] = self.instance_baselines[new_index].clone();
 
-                    for entity in message.entities.iter() {
+                    for entity in ent_message.entities.iter() {
                         self.instance_baselines[new_index]
                             .insert(entity.entity_index, entity.props.clone());
                     }
                 }
 
-                for removed in message.removed_entities.iter() {
+                for removed in ent_message.removed_entities.iter() {
                     self.entity_classes.remove(removed);
                 }
 
-                for entity in message.entities.iter() {
+                for entity in ent_message.entities.iter() {
                     if entity.pvs == PVS::Delete {
                         self.entity_classes.remove(&entity.entity_index);
                     }
                     self.entity_classes
                         .insert(entity.entity_index, Rc::clone(&entity.server_class));
                 }
+                Some(Message::PacketEntities(ent_message))
             }
-            _ => {}
+            _ => Some(message),
         }
     }
 
-    fn handle_string_entry(&mut self, table: &String, _index: usize, entry: &StringTableEntry) {
+    pub fn handle_string_entry(&mut self, table: &String, _index: usize, entry: &StringTableEntry) {
         match table.as_str() {
             "instancebaseline" => {
                 if let (Some(extra), Ok(class_id)) = (&entry.extra_data, entry.text().parse()) {
@@ -173,8 +173,4 @@ impl MessageHandler for ParserState {
             _ => {}
         }
     }
-
-    fn get_output(self, _state: ParserState) -> Self {
-        self
-    }
 }

src/demo/sendprop.rs

@@ -1,6 +1,7 @@
 use bitstream_reader::{BitRead, LittleEndian};
 use enumflags2::BitFlags;
 use enumflags2_derive::EnumFlags;
+use serde::{Deserialize, Serialize};
 
 use crate::{MalformedDemoError, Parse, ParseError, ReadResult, Result, Stream};
 
@@ -14,7 +15,9 @@ use std::convert::TryInto;
 use std::fmt;
 use std::rc::Rc;
 
-#[derive(BitRead, PartialEq, Eq, Hash, Debug, Display, Clone)]
+#[derive(
+    BitRead, PartialEq, Eq, Hash, Debug, Display, Clone, Serialize, Deserialize, Ord, PartialOrd,
+)]
 pub struct SendPropName(Rc<String>);
 
 impl PartialEq<&str> for SendPropName {
@@ -287,16 +290,66 @@ impl BitRead<LittleEndian> for SendPropFlags {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(untagged)]
 pub enum SendPropValue {
     Vector(Vector),
     VectorXY(VectorXY),
-    Integer(i32),
+    Integer(i64),
     Float(f32),
     String(String),
     Array(Vec<SendPropValue>),
 }
 
+impl PartialEq for SendPropValue {
+    fn eq(&self, other: &Self) -> bool {
+        // allow comparing some "compatible" types
+        match (self, other) {
+            (SendPropValue::Vector(value1), SendPropValue::Vector(value2)) => value1 == value2,
+            (SendPropValue::VectorXY(value1), SendPropValue::VectorXY(value2)) => value1 == value2,
+            (SendPropValue::Integer(value1), SendPropValue::Integer(value2)) => value1 == value2,
+            (SendPropValue::Float(value1), SendPropValue::Float(value2)) => value1 - value2 < 0.001,
+            (SendPropValue::String(value1), SendPropValue::String(value2)) => value1 == value2,
+            (SendPropValue::Array(value1), SendPropValue::Array(value2)) => value1 == value2,
+            (SendPropValue::Integer(value1), SendPropValue::Float(value2)) => {
+                *value1 as f64 == *value2 as f64
+            }
+            (SendPropValue::Float(value1), SendPropValue::Integer(value2)) => {
+                *value1 as f64 == *value2 as f64
+            }
+            (SendPropValue::Vector(value1), SendPropValue::VectorXY(value2)) => {
+                value1.x == value2.x && value1.y == value2.y && value1.z == 0.0
+            }
+            (SendPropValue::VectorXY(value1), SendPropValue::Vector(value2)) => {
+                value1.x == value2.x && value1.y == value2.y && value2.z == 0.0
+            }
+            (SendPropValue::Vector(value1), SendPropValue::Array(value2)) if value2.len() == 3 => {
+                SendPropValue::Float(value1.x) == value2[0]
+                    && SendPropValue::Float(value1.y) == value2[1]
+                    && SendPropValue::Float(value1.z) == value2[2]
+            }
+            (SendPropValue::Array(value1), SendPropValue::Vector(value2)) if value1.len() == 3 => {
+                SendPropValue::Float(value2.x) == value1[0]
+                    && SendPropValue::Float(value2.y) == value1[1]
+                    && SendPropValue::Float(value2.z) == value1[2]
+            }
+            (SendPropValue::VectorXY(value1), SendPropValue::Array(value2))
+                if value2.len() == 2 =>
+            {
+                SendPropValue::Float(value1.x) == value2[0]
+                    && SendPropValue::Float(value1.y) == value2[1]
+            }
+            (SendPropValue::Array(value1), SendPropValue::VectorXY(value2))
+                if value1.len() == 2 =>
+            {
+                SendPropValue::Float(value2.x) == value1[0]
+                    && SendPropValue::Float(value2.y) == value1[1]
+            }
+            _ => false,
+        }
+    }
+}
+
 impl fmt::Display for SendPropValue {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -331,16 +384,17 @@ impl SendPropValue {
         }
     }
 
-    fn read_int(stream: &mut Stream, definition: &SendPropDefinition) -> Result<i32> {
+    fn read_int(stream: &mut Stream, definition: &SendPropDefinition) -> Result<i64> {
         if definition.flags.contains(SendPropFlag::NormalVarInt) {
             read_var_int(stream, !definition.flags.contains(SendPropFlag::Unsigned))
                 .map_err(ParseError::from)
+                .map(|int| int as i64)
         } else {
             if definition.flags.contains(SendPropFlag::Unsigned) {
                 let unsigned: u32 =
                     stream.read_sized(definition.bit_count.unwrap_or(32) as usize)?;
-                const MAX: u32 = std::i32::MAX as u32;
+                //const MAX: u32 = std::i32::MAX as u32;
-                Ok((unsigned & MAX) as i32)
+                Ok(unsigned as i64)
             } else {
                 stream
                     .read_int(definition.bit_count.unwrap_or(32) as usize)
@@ -418,7 +472,8 @@ impl SendPropValue {
                 .low_value
                 .ok_or(MalformedSendPropDefinitionError::UnsizedFloat)?;
             let raw: u32 = stream.read_int(bit_count as usize)?;
-            let percentage = (raw as f32) * get_frac_factor(bit_count as usize);
+            // is this -1 correct?, it is consistent with the js version but seems weird
+            let percentage = (raw as f32) / ((1 << bit_count) as f32 - 1.0);
             Ok(low + ((high - low) * percentage))
         }
     }
@@ -426,6 +481,12 @@ impl SendPropValue {
 
 impl From<i32> for SendPropValue {
     fn from(value: i32) -> Self {
+        SendPropValue::Integer(value as i64)
+    }
+}
+
+impl From<i64> for SendPropValue {
+    fn from(value: i64) -> Self {
         SendPropValue::Integer(value)
     }
 }

src/demo/vector.rs

@@ -2,9 +2,7 @@ use bitstream_reader::{BitRead, BitSize};
 use parse_display::Display;
 use serde::{Deserialize, Serialize};
 
-#[derive(
+#[derive(BitRead, BitSize, Debug, Clone, Copy, Default, Serialize, Deserialize, Display)]
-    BitRead, BitSize, Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Display,
-)]
 #[display("({x}, {y}, {z})")]
 pub struct Vector {
     pub x: f32,
@@ -12,11 +10,21 @@ pub struct Vector {
     pub z: f32,
 }
 
-#[derive(
+impl PartialEq for Vector {
-    BitRead, BitSize, Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Display,
+    fn eq(&self, other: &Self) -> bool {
-)]
+        (self.x - other.x < 0.001) && (self.y - other.y < 0.001) && (self.z - other.z < 0.001)
+    }
+}
+
+#[derive(BitRead, BitSize, Debug, Clone, Copy, Default, Serialize, Deserialize, Display)]
 #[display("({x}, {y})")]
 pub struct VectorXY {
     pub x: f32,
     pub y: f32,
 }
+
+impl PartialEq for VectorXY {
+    fn eq(&self, other: &Self) -> bool {
+        (self.x - other.x < 0.001) && (self.y - other.y < 0.001)
+    }
+}

tests/entity.rs

@@ -0,0 +1,190 @@
+#![allow(dead_code)]
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+
+use pretty_assertions::assert_eq;
+use serde::{Deserialize, Serialize};
+use serde_repr::{Deserialize_repr, Serialize_repr};
+use std::fs::{self, File};
+
+use std::collections::{HashMap, HashSet};
+use std::io::{BufRead, BufReader};
+use std::rc::Rc;
+use tf_demo_parser::demo::message::packetentities::{EntityId, PacketEntity, PVS};
+use tf_demo_parser::demo::message::Message;
+use tf_demo_parser::demo::packet::datatable::{
+    ParseSendTable, SendTableName, ServerClass, ServerClassName,
+};
+use tf_demo_parser::demo::packet::stringtable::StringTableEntry;
+use tf_demo_parser::demo::parser::MessageHandler;
+use tf_demo_parser::demo::sendprop::{SendPropDefinition, SendPropName, SendPropValue};
+use tf_demo_parser::{Demo, DemoParser, MatchState, MessageType, MessageTypeAnalyser, ParserState};
+
+/// Compatible serialization with the js parser entity dumps
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
+#[repr(u8)]
+pub enum PVSCompat {
+    Preserve = 0,
+    Leave = 2,
+    Enter = 1,
+    Delete = 6,
+}
+
+impl From<PVS> for PVSCompat {
+    fn from(pvs: PVS) -> Self {
+        match pvs {
+            PVS::Preserve => PVSCompat::Preserve,
+            PVS::Leave => PVSCompat::Leave,
+            PVS::Enter => PVSCompat::Enter,
+            PVS::Delete => PVSCompat::Delete,
+        }
+    }
+}
+
+#[derive(PartialEq, Clone, Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct EntityDump {
+    tick: u32,
+    server_class: ServerClassName,
+    id: EntityId,
+    props: HashMap<SendPropName, SendPropValue>,
+    pvs: PVSCompat,
+}
+
+impl EntityDump {
+    pub fn from_entity(entity: PacketEntity, tick: u32) -> Self {
+        let id = entity.entity_index;
+        EntityDump {
+            tick,
+            server_class: entity.server_class.name.clone(),
+            id: entity.entity_index,
+            props: entity
+                .props
+                .into_iter()
+                .map(|prop| (prop.definition.name.clone(), prop.value))
+                .collect(),
+            pvs: entity.pvs.into(),
+        }
+    }
+}
+
+struct EntityDumper {
+    entities: Vec<EntityDump>,
+}
+
+impl EntityDumper {
+    pub fn new() -> Self {
+        EntityDumper {
+            entities: Vec::with_capacity(128),
+        }
+    }
+}
+
+impl MessageHandler for EntityDumper {
+    type Output = Vec<EntityDump>;
+
+    fn does_handle(message_type: MessageType) -> bool {
+        match message_type {
+            MessageType::PacketEntities => true,
+            _ => false,
+        }
+    }
+
+    fn handle_message(&mut self, message: Message, tick: u32) {
+        match message {
+            Message::PacketEntities(entity_message) => self.entities.extend(
+                entity_message
+                    .entities
+                    .into_iter()
+                    .map(|entity| EntityDump::from_entity(entity, tick)),
+            ),
+            _ => {}
+        }
+    }
+
+    fn handle_string_entry(&mut self, table: &String, _index: usize, entry: &StringTableEntry) {}
+
+    fn get_output(self, _state: ParserState) -> Self::Output {
+        self.entities
+    }
+}
+
+fn entity_test(input_file: &str, snapshot_file: &str) {
+    let file = fs::read(input_file).expect("Unable to read file");
+    let demo = Demo::new(file);
+    let (_, entities) =
+        DemoParser::parse_with_analyser(demo.get_stream(), EntityDumper::new()).unwrap();
+
+    let json_file = File::open(snapshot_file).expect("Unable to read file");
+    let mut reader = BufReader::new(json_file);
+    let mut buffer = String::new();
+
+    let mut expected = Vec::with_capacity(128);
+
+    while reader.read_line(&mut buffer).expect("failed to read line") > 0 {
+        let entity: EntityDump =
+            serde_json::from_str(buffer.trim_end()).expect("failed to parse json");
+        expected.push(entity);
+        buffer.clear();
+    }
+
+    assert_eq!(expected.len(), entities.len());
+
+    let entity_ids: Vec<_> = entities.iter().map(|entity| entity.id).collect();
+    let expected_ids: Vec<_> = expected.iter().map(|entity| entity.id).collect();
+
+    assert_eq!(expected_ids, entity_ids);
+
+    for (i, (expected_entity, entity)) in expected.into_iter().zip(entities.into_iter()).enumerate()
+    {
+        assert_eq!(
+            expected_entity.tick, entity.tick,
+            "Failed comparing entity {}",
+            i
+        );
+        assert_eq!(
+            expected_entity.id, entity.id,
+            "Failed comparing entity {}",
+            i
+        );
+        assert_eq!(
+            expected_entity.server_class, entity.server_class,
+            "Failed comparing entity {}",
+            i
+        );
+        assert_eq!(
+            expected_entity.pvs, entity.pvs,
+            "Failed comparing entity {}",
+            i
+        );
+        let mut prop_names: Vec<_> = entity.props.keys().collect();
+        let mut expected_prop_names: Vec<_> = expected_entity.props.keys().collect();
+        prop_names.sort();
+        expected_prop_names.sort();
+
+        assert_eq!(
+            expected_prop_names, prop_names,
+            "Failed comparing entity {}",
+            i
+        );
+
+        for prop_name in expected_prop_names {
+            assert_eq!(
+                expected_entity.props.get(prop_name),
+                entity.props.get(prop_name),
+                "Failed comparing entity {} prop {}",
+                i,
+                prop_name
+            );
+        }
+
+        assert_eq!(expected_entity, entity, "Failed comparing entity {}", i);
+    }
+}
+
+#[test]
+fn entity_test_short() {
+    better_panic::install();
+
+    entity_test("data/small.dem", "data/small_entities.json");
+}