first prop flattening version

Cargo.lock

@@ -184,14 +184,6 @@ name = "fuchsia-cprng"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "heck"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "itoa"
 version = "0.4.4"
@@ -531,22 +523,6 @@ dependencies = [
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "strum"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "strum_macros"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "syn"
 version = "0.11.11"
@@ -609,8 +585,6 @@ dependencies = [
  "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_repr 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "strum 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "strum_macros 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -626,11 +600,6 @@ name = "ucd-util"
 version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.5"
@@ -694,7 +663,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum enumflags2 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8c30a444d141611c4826a1a2e7eba9a494458edf14754945fa37c9c20f6f7563"
 "checksum enumflags2_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e015b3dfedc096cb55cdc5d022d6b4e6b94547212fb94ad2d9ece20bcd88fe3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
 "checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14"
 "checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319"
@@ -737,8 +705,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum serde_repr 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "29a734c298df0346c4cd5919595981c266dabbf12dc747c85e1a95e96077a52b"
 "checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7"
 "checksum snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "95d697d63d44ad8b78b8d235bf85b34022a78af292c8918527c5f0cffdde7f43"
-"checksum strum 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d1c33039533f051704951680f1adfd468fd37ac46816ded0d9ee068e60f05f"
-"checksum strum_macros 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "47cd23f5c7dee395a00fa20135e2ec0fffcdfa151c56182966d7a3261343432e"
 "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
 "checksum syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d960b829a55e56db167e861ddb43602c003c7be0bee1d345021703fac2fb7c"
 "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
@@ -746,7 +712,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86"
-"checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9"
 "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
 "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"

Cargo.toml

@@ -19,12 +19,10 @@ num-traits = "0.2"
 enumflags2 = "0.5"
 enumflags2_derive = "0.5"
 snap = "0.2"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 serde_repr = "0.1"
 better-panic = "0.1"
-strum = "0.15"
-strum_macros = "0.15"
 
 [dev-dependencies]
 pretty_assertions = "0.6"

pgo_build.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+rm -rf /tmp/pgo-data
+
+mkdir -p /tmp/demos
+
+function download_demo {
+    if [ ! -f "/tmp/demos/$1.dem" ]; then
+        URL=$(curl -s https://api.demos.tf/demos/$1 | grep -oP "\"(https:.*?\.dem)\"")
+        URL="${URL//\\/}"
+        URL="${URL//\"/}"
+        echo "Downloading demo $1 from $URL"
+        wget -q $URL -O /tmp/demos/$1.dem
+    fi
+}
+
+function profile_demo {
+    echo "/tmp/demos/$1.dem"
+    ./target/release/parse_demo "/tmp/demos/$1.dem" > /dev/null
+}
+
+for i in $(seq 283600 283700); do download_demo $i; done
+
+RUSTFLAGS="-Cprofile-generate=/tmp/pgo-data" \
+    cargo +nightly build --release --target=x86_64-unknown-linux-gnu
+
+echo "Profiling demos"
+
+for i in $(seq 283600 283700); do profile_demo $i; done
+
+#llvm-profdata merge -o /tmp/pgo-data/merged.profdata /tmp/pgo-data
+
+#RUSTFLAGS="-Cprofile-use=/tmp/pgo-data/merged.profdata" \
+#    cargo +nightly build --release

src/demo/packet/datatable.rs

@@ -2,7 +2,9 @@ use bitstream_reader::{BitRead, LittleEndian};
 
 use crate::demo::sendprop::{SendPropDefinition, SendPropFlag, SendPropType};
 use crate::{Parse, ParseError, ParserState, Result, Stream, ReadResult};
-use strum_macros::{EnumString, Display};
+use std::fmt;
+use serde::{Serialize, Deserialize};
+use std::rc::Rc;
 
 #[derive(BitRead, Debug)]
 pub struct ServerClass {
@@ -11,15 +13,24 @@ pub struct ServerClass {
     pub data_table: String,
 }
 
-#[derive(PartialEq, Eq, Hash, Debug, Clone, EnumString, Display)]
+#[derive(PartialEq, Eq, Hash, Debug, Serialize, Deserialize)]
-pub enum SendTableName {
+pub struct SendTableName(Rc<String>);
-    #[strum(default = "true")]
+
-    Other(String)
+impl Clone for SendTableName {
+    fn clone(&self) -> Self {
+        SendTableName(Rc::clone(&self.0))
+    }
+}
+
+impl fmt::Display for SendTableName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
 }
 
 impl From<String> for SendTableName {
     fn from(value: String) -> Self {
-        value.parse().unwrap()
+        Self(Rc::new(value))
     }
 }
 
@@ -40,7 +51,8 @@ pub struct SendTable {
 impl Parse for SendTable {
     fn parse(stream: &mut Stream, _state: &ParserState) -> Result<Self> {
         let needs_decoder = stream.read()?;
-        let name: String = stream.read()?;
+        let raw_name: String = stream.read()?;
+        let name: SendTableName = raw_name.into();
         let prop_count = stream.read_int(10)?;
 
         let mut array_element_prop = None;
@@ -48,7 +60,7 @@ impl Parse for SendTable {
 
         for _ in 0..prop_count {
             let prop: SendPropDefinition =
-                SendPropDefinition::read(stream)?;
+                SendPropDefinition::read(stream, name.clone())?;
             if prop.flags.contains(SendPropFlag::InsideArray) {
                 if array_element_prop.is_some()
                     || prop.flags.contains(SendPropFlag::ChangesOften)
@@ -72,7 +84,7 @@ impl Parse for SendTable {
         }
 
         Ok(SendTable {
-            name: SendTableName::from(name),
+            name,
             flattened_props: None,
             needs_decoder,
             props,
@@ -80,35 +92,88 @@ impl Parse for SendTable {
     }
 }
 
-//impl SendTable {
+impl SendTable {
-//    fn get_excludes(&self) -> ExcludesIterator {
+    pub fn flatten_props(&self, tables: &[SendTable]) -> Vec<SendPropDefinition> {
-//        ExcludesIterator {
+        let mut flat = Vec::new();
-//            table: self,
+        self.get_all_props(tables, &self.get_excludes(tables), &mut flat);
-//            index: 0,
+
-//        }
+        // sort often changed props before the others
-//    }
+        let mut start = 0;
-//}
+        for i in 0..flat.len() {
-//
+            if flat[i].flags.contains(SendPropFlag::ChangesOften) {
-//struct ExcludesIterator<'a> {
+                if i != start {
-//    table: &'a SendTable,
+                    flat.swap(i, start);
-//    index: usize,
+                }
-//}
+                start += 1;
-//
+            }
-//impl<'a> Iterator for ExcludesIterator<'a> {
+        }
-//    type Item = &'a SendPropDefinition;
+
-//
+        flat
-//    fn next(&mut self) -> Option<Self::Item> {
+    }
-//        while let Some(prop) = self.table.props.get(index) {
+
-//            index += 1;
+    fn get_excludes<'a>(&'a self, tables: &'a [SendTable]) -> Vec<Exclude<'a>> {
-//            if prop.flags.contains(SendPropFlag::Exclude) {
+        let mut excludes = Vec::new();
-//                return Some(prop);
+
-//            } else if prop.prop_type == SendPropType::DataTable {
+        for prop in self.props.iter() {
-//                if let Some(table)
+            if prop.is_exclude() && prop.table_name.is_some() {
-//            }
+                excludes.push(Exclude {
-//        }
+                    table: prop.table_name.as_ref().unwrap(),
-//        None
+                    prop: &prop.name,
-//    }
+                })
-//}
+            } else if prop.prop_type == SendPropType::DataTable {
+                if let Some(table) = prop.get_data_table(tables) {
+                    excludes.extend_from_slice(&table.get_excludes(tables));
+                }
+            }
+        }
+
+        excludes
+    }
+
+    // TODO: below is a direct port from the js which is a direct port from C++ and not very optimal
+    fn get_all_props(&self, tables: &[SendTable], excludes: &[Exclude], props: &mut Vec<SendPropDefinition>) {
+        let mut local_props = Vec::new();
+
+        self.get_all_props_iterator_props(tables, excludes, &mut local_props, props);
+        props.extend_from_slice(&local_props);
+    }
+
+    fn get_all_props_iterator_props(&self, tables: &[SendTable], excludes: &[Exclude], props: &mut Vec<SendPropDefinition>, child_props: &mut Vec<SendPropDefinition>) {
+        for prop in self.props.iter() {
+            if prop.is_exclude() {
+                continue;
+            }
+
+            if excludes.iter().any(|exclude| exclude.matches(prop)) {
+                continue;
+            }
+
+            if prop.prop_type == SendPropType::DataTable {
+                if let Some(table) = prop.get_data_table(tables) {
+                    if prop.flags.contains(SendPropFlag::Collapsible) {
+                        table.get_all_props_iterator_props(tables, excludes, props, child_props);
+                    } else {
+                        table.get_all_props(tables, excludes, child_props);
+                    }
+                }
+            } else {
+                props.push(prop.clone());
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
+struct Exclude<'a> {
+    table: &'a SendTableName,
+    prop: &'a str,
+}
+
+impl<'a> Exclude<'a> {
+    pub fn matches(&self, prop: &SendPropDefinition) -> bool {
+        self.prop == prop.name && self.table == &prop.owner_table
+    }
+}
 
 #[derive(Debug)]
 pub struct DataTablePacket {

src/demo/parser/mod.rs

@@ -5,7 +5,7 @@ use crate::demo::header::Header;
 use crate::demo::packet::Packet;
 use crate::demo::parser::analyser::Analyser;
 pub use crate::demo::parser::analyser::MatchState;
-use crate::demo::parser::handler::{DemoHandler, MessageHandler};
+pub use crate::demo::parser::handler::{DemoHandler, MessageHandler};
 pub use crate::demo::parser::state::ParserState;
 use crate::Stream;
 pub use self::messagetypeanalyser::MessageTypeAnalyser;

src/demo/sendprop.rs

@@ -8,12 +8,13 @@ use super::packet::datatable::SendTable;
 use super::vector::{Vector, VectorXY};
 use crate::demo::packet::datatable::SendTableName;
 
-#[derive(Debug)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct SendPropDefinition {
     pub prop_type: SendPropType,
     pub name: String,
     pub flags: SendPropFlags,
     pub table_name: Option<SendTableName>,
+    pub owner_table: SendTableName,
     pub low_value: Option<f32>,
     pub high_value: Option<f32>,
     pub bit_count: Option<u32>,
@@ -33,17 +34,19 @@ impl SendPropDefinition {
             bit_count: self.bit_count,
             element_count: self.element_count,
             array_property: Some(Box::new(array_property)),
+            owner_table: self.owner_table,
         }
     }
 
+    /// Get the refered data table
+    ///
+    /// Note that this is not the owner table
     pub fn get_data_table<'a>(&self, tables: &'a [SendTable]) -> Option<&'a SendTable> {
         self.table_name.as_ref()
             .and_then(|name| tables.iter().find(|table| table.name == *name))
     }
-}
 
-impl BitRead<LittleEndian> for SendPropDefinition {
+    pub fn read(stream: &mut Stream, owner_table: SendTableName) -> ReadResult<Self> {
-    fn read(stream: &mut Stream) -> ReadResult<Self> {
         let prop_type = SendPropType::read(stream)?;
         let name = stream.read_string(None)?;
         let flags = SendPropFlags::read(stream)?;
@@ -86,8 +89,13 @@ impl BitRead<LittleEndian> for SendPropDefinition {
             bit_count,
             element_count,
             array_property: None,
+            owner_table,
         })
     }
+
+    pub fn is_exclude(&self) -> bool {
+        self.flags.contains(SendPropFlag::Exclude)
+    }
 }
 
 #[derive(BitRead, Copy, Clone, PartialEq, Debug)]
@@ -165,7 +173,7 @@ pub enum SendPropFlag {
     NormalVarInt = 32, //(1 << 5)
 }
 
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, PartialEq)]
 pub struct SendPropFlags(BitFlags<SendPropFlag>);
 
 impl SendPropFlags {

src/lib.rs

@@ -1,3 +1,7 @@
+#![allow(dead_code)]
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+
 pub use bitstream_reader::Result as ReadResult;
 
 pub use crate::demo::{
@@ -8,4 +12,4 @@ pub use crate::demo::{
     Stream,
 };
 
-mod demo;
+pub mod demo;

tests/sendprops.rs

@@ -0,0 +1,65 @@
+#![allow(dead_code)]
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+
+use std::fs;
+use pretty_assertions::assert_eq;
+
+use tf_demo_parser::{Demo, DemoParser, MatchState, MessageTypeAnalyser, MessageType, ParserState};
+use tf_demo_parser::demo::packet::datatable::{SendTable, SendTableName};
+use tf_demo_parser::demo::packet::stringtable::StringTableEntry;
+use tf_demo_parser::demo::message::Message;
+use tf_demo_parser::demo::sendprop::SendPropDefinition;
+use std::collections::{HashMap, HashSet};
+use tf_demo_parser::demo::parser::MessageHandler;
+
+pub struct SendPropAnalyser;
+
+impl MessageHandler for SendPropAnalyser {
+    type Output = Vec<SendTable>;
+
+    fn does_handle(message_type: MessageType) -> bool {
+        false
+    }
+
+    fn handle_message(&mut self, message: Message, tick: u32) {}
+
+    fn handle_string_entry(&mut self, table: &String, _index: usize, entry: &StringTableEntry) {}
+
+    fn get_output(self, state: ParserState) -> Self::Output {
+        state.send_tables.into_iter().map(|(_k, v)| v).collect()
+    }
+}
+
+fn flatten_test(input_file: &str, snapshot_file: &str) {
+    let file = fs::read(input_file).expect("Unable to read file");
+    let demo = Demo::new(file);
+    let (_, send_tables) = DemoParser::parse_with_analyser(demo.get_stream(), SendPropAnalyser).unwrap();
+    let flat_props: HashMap<SendTableName, Vec<String>> = send_tables.iter()
+        .map(|table| {
+            (
+                table.name.clone(),
+                table.flatten_props(&send_tables)
+                    .into_iter()
+                    .map(|prop| format!("{}.{}", prop.owner_table, prop.name))
+                    .collect()
+            )
+        })
+        .collect();
+
+    let expected: HashMap<SendTableName, Vec<String>> = serde_json::from_slice(fs::read(snapshot_file).expect("Unable to read file").as_slice()).unwrap();
+
+    let expected_tables: HashSet<_> = expected.keys().collect();
+    let actual_tables: HashSet<_> = flat_props.keys().collect();
+
+    assert_eq!(expected_tables, actual_tables);
+    for table in expected_tables {
+        dbg!(table);
+        assert_eq!(expected[table], flat_props[table]);
+    }
+}
+
+#[test]
+fn sendprop_test_gully() {
+    flatten_test("data/gully.dem", "data/gully_props.json");
+}