stringtables

Cargo.lock

@@ -1,5 +1,10 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "arraydeque"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "bitstream_reader"
 version = "0.3.1"
@@ -108,6 +113,7 @@ dependencies = [
 name = "tf-demo-parser"
 version = "0.1.0"
 dependencies = [
+ "arraydeque 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bitstream_reader 0.3.1",
  "enum-primitive-derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "enumflags2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -126,6 +132,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [metadata]
+"checksum arraydeque 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e300327073b806ffc81fccb228b2d4131ac7ef1b1a015f7b0c399c7f886cacc6"
 "checksum enum-primitive-derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2b90e520ec62c1864c8c78d637acbfe8baf5f63240f2fb8165b8325c07812dd"
 "checksum enumflags2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "801303a673e02d2110a196d70e4a807f0ff0a68ce1060eebc1e6a0d598db65e8"
 "checksum enumflags2_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e015b3dfedc096cb55cdc5d022d6b4e6b94547212fb94ad2d9ece20bcd88fe3"

Cargo.toml

@@ -17,3 +17,4 @@ enum-primitive-derive = "0.1.2"
 num-traits = "0.2"
 enumflags2 = "0.5"
 enumflags2_derive = "0.5"
+arraydeque = "0.4"

src/demo/header.rs

@@ -1,4 +1,4 @@
-use bitstream_reader::{BitRead};
+use bitstream_reader::BitRead;
 
 #[derive(BitRead, Debug, PartialEq)]
 pub struct Header {

src/demo/message/classinfo.rs

@@ -0,0 +1,35 @@
+use bitstream_reader::{BitRead, BitReadSized, LittleEndian};
+
+use crate::{ReadResult, Stream};
+
+#[derive(BitReadSized)]
+pub struct ClassInfoEntry {
+    #[size = "input_size"]
+    class_id: u16,
+    class_name: String,
+    table_name: String,
+}
+
+pub struct ClassInfoMessage {
+    count: u16,
+    create: bool,
+    entries: Vec<ClassInfoEntry>,
+}
+
+impl BitRead<LittleEndian> for ClassInfoMessage {
+    fn read(stream: &mut Stream) -> ReadResult<Self> {
+        let count: u16 = stream.read()?;
+        let create = stream.read()?;
+        let mut entries = Vec::with_capacity(count as usize);
+        let bits = 16 - count.leading_zeros();
+        for _ in 0..count {
+            entries.push(stream.read_sized(bits as usize)?);
+        }
+
+        Ok(ClassInfoMessage {
+            count,
+            create,
+            entries,
+        })
+    }
+}

src/demo/message/generated.rs

@@ -1,5 +1,4 @@
 /// Messages that consists only of primitives and string and can be derived
-
 use bitstream_reader::BitRead;
 use std::collections::HashMap;
 
@@ -19,7 +18,7 @@ pub struct NetTickMessage {
 
 #[derive(BitRead, Debug)]
 pub struct StringCmdMessage {
-    pub command: String
+    pub command: String,
 }
 
 #[derive(BitRead, Debug)]
@@ -30,7 +29,7 @@ pub struct SigOnStateMessage {
 
 #[derive(BitRead, Debug)]
 pub struct PrintMessage {
-    pub value: String
+    pub value: String,
 }
 
 #[derive(BitRead, Debug)]
@@ -56,13 +55,13 @@ pub struct ServerInfoMessage {
 
 #[derive(BitRead, Debug)]
 pub struct SetPauseMessage {
-    pub pause: bool
+    pub pause: bool,
 }
 
 #[derive(BitRead, Debug)]
 pub struct SetViewMessage {
     #[size = 11]
-    pub index: u16
+    pub index: u16,
 }
 
 #[derive(BitRead, Debug)]
@@ -76,10 +75,9 @@ pub struct FixAngleMessage {
 #[derive(BitRead, Debug)]
 pub struct PreFetchMessage {
     #[size = 14]
-    pub index: u16
+    pub index: u16,
 }
 
-
 #[derive(BitRead, Debug)]
 pub struct GetCvarMessage {
     pub cookie: u32,
@@ -89,5 +87,5 @@ pub struct GetCvarMessage {
 #[derive(BitRead, Debug)]
 pub struct SetConVarMessage {
     #[size_bits = 8]
-    vars: HashMap<String, String>
+    vars: HashMap<String, String>,
 }

src/demo/message/mod.rs

@@ -5,7 +5,9 @@ pub use generated::*;
 
 use crate::{Parse, ParseError, ParserState, Result, Stream};
 
+mod classinfo;
 mod generated;
+mod stringtable;
 
 #[derive(Primitive, Debug)]
 pub enum MessageType {

src/demo/message/stringtable.rs

@@ -0,0 +1,162 @@
+use std::collections::HashMap;
+
+use arraydeque::{ArrayDeque, Wrapping};
+use bitstream_reader::{BitRead, BitReadSized, LittleEndian};
+
+use crate::{ReadResult, Stream};
+use crate::demo::packet::stringtable::{ExtraData, FixedUserdataSize, StringTable, StringTableEntry};
+
+pub struct CreateStringTableMessage {
+    pub table: StringTable,
+}
+
+struct StringTableMeta {
+    pub max_entries: u16,
+    pub fixed_userdata_size: Option<FixedUserdataSize>,
+}
+
+impl From<&StringTable> for StringTableMeta {
+    fn from(table: &StringTable) -> Self {
+        StringTableMeta {
+            max_entries: table.max_entries,
+            fixed_userdata_size: table.fixed_userdata_size,
+        }
+    }
+}
+
+impl BitRead<LittleEndian> for CreateStringTableMessage {
+    fn read(stream: &mut Stream) -> ReadResult<Self> {
+        let name = stream.read()?;
+        let max_entries: u16 = stream.read()?;
+        let encode_bits = 16 - max_entries.leading_zeros();
+        let entity_count: u16 = stream.read_sized(encode_bits as usize + 1)?;
+        let bit_count = read_var_int(stream)?;
+
+        let fixed_userdata_size = stream.read()?;
+
+        let compressed = stream.read()?;
+
+        let table_data = stream.read_bits(bit_count as usize)?;
+
+        if compressed {
+            unimplemented!("TODO: SNAP decoding");
+        }
+
+        let table_meta = StringTableMeta {
+            max_entries,
+            fixed_userdata_size,
+        };
+
+        let entries = parse_string_table_entries(stream, &table_meta, entity_count, Vec::new())?;
+        let mut entries: Vec<(u16, StringTableEntry)> = entries.into_iter().collect();
+
+        // verify that there are no holes in our indexes
+        entries.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
+        let last_index = entries.last().map(|(index, _)| *index).unwrap_or(0u16) as usize;
+        if last_index != entries.len() {
+            panic!("there should be no holes when reading CreateStringTable message");
+        }
+        let table_entries = entries.into_iter().map(|(_, entry)| entry).collect();
+        let table = StringTable {
+            entries: table_entries,
+            max_entries,
+            fixed_userdata_size,
+            client_entries: None,
+            compressed,
+            name,
+        };
+        Ok(CreateStringTableMessage {
+            table
+        })
+    }
+}
+
+fn parse_string_table_entries(
+    stream: &mut Stream,
+    table_meta: &StringTableMeta,
+    entry_count: u16,
+    existing_entries: Vec<StringTableEntry>,
+) -> ReadResult<HashMap<u16, StringTableEntry>> {
+    let entry_bits = 16 - table_meta.max_entries.leading_zeros();
+    let mut entries = HashMap::new();
+
+    let mut last_entry: i16 = -1;
+    let mut history: ArrayDeque<[String; 32], Wrapping> = ArrayDeque::new();
+
+    for i in 0..entry_count {
+        let index = if stream.read()? {
+            (last_entry + 1) as u16
+        } else {
+            stream.read_sized(entry_bits as usize)?
+        };
+
+        let value = if stream.read()? { // set value
+            if stream.read()? { // reuse from history
+                let index: u16 = stream.read_sized(5)?;
+                let bytes_to_copy: u32 = stream.read_sized(5)?;
+                let rest_of_string: String = stream.read()?;
+
+                Some(match history.get(index as usize) {
+                    Some(text) => String::from_utf8({
+                        text.bytes().take(bytes_to_copy as usize).chain(rest_of_string.bytes()).collect()
+                    })?,
+                    None => rest_of_string // best guess, happens in some pov demos but only for unimportant tables it seems
+                })
+            } else {
+                Some(stream.read()?)
+            }
+        } else {
+            None
+        };
+
+        let user_data = if stream.read()? {
+            Some(match table_meta.fixed_userdata_size {
+                Some(size) => stream.read_bits(size.bits as usize)?,
+                None => {
+                    let bytes: u16 = stream.read_sized(14)?;
+                    stream.read_bits(bytes as usize)?
+                }
+            })
+        } else {
+            None
+        }.map(|stream| ExtraData {
+            len: stream.bit_len() as u16 / 8,
+            data: stream,
+        });
+
+        let entry = match existing_entries.get(index as usize) {
+            Some(existing_entry) => {
+                let new_user_data = user_data.or_else(|| existing_entry.extra_data.clone());
+                let new_value = value.unwrap_or_else(|| existing_entry.text.clone());
+                StringTableEntry {
+                    text: new_value,
+                    extra_data: new_user_data,
+                }
+            }
+            None => StringTableEntry {
+                text: value.unwrap_or_default(),
+                extra_data: user_data,
+            }
+        };
+        let text = entry.text.clone();
+        entries.insert(index, entry);
+        unsafe {
+            history.push_back(text);
+        }
+    }
+
+    Ok(entries)
+}
+
+fn read_var_int(stream: &mut Stream) -> ReadResult<u32> {
+    let mut result: u32 = 0;
+    for i in (0..35u32).step_by(7) {
+        let byte: u8 = stream.read()?;
+        result |= (byte as u32 & 0x7F) << i;
+
+        if (byte >> 7) == 0 {
+            break;
+        }
+    }
+    Ok(result)
+}

src/demo/packet/consolecmd.rs

@@ -1,6 +1,6 @@
 use bitstream_reader::{BitRead, LittleEndian};
 
-use crate::{Stream, ReadResult};
+use crate::{ReadResult, Stream};
 
 #[derive(Debug)]
 pub struct ConsoleCmdPacket {

src/demo/packet/datatable.rs

@@ -1,7 +1,7 @@
 use bitstream_reader::BitRead;
 
-use crate::{Parse, ParseError, ParserState, Result, Stream};
 use crate::demo::sendprop::{SendPropDefinition, SendPropFlag, SendPropType};
+use crate::{Parse, ParseError, ParserState, Result, Stream};
 
 #[derive(BitRead, Debug)]
 pub struct ServerClass {
@@ -34,7 +34,7 @@ impl Parse for DataTablePacket {
 
         let mut tables = vec![];
         while packet_data.read_bool()? {
-            let needs_decoder = packet_data.read_bool()?;
+            let needs_decoder = packet_data.read()?;
             let name: String = packet_data.read()?;
             let prop_count = packet_data.read_int(10)?;
 

src/demo/packet/mod.rs

@@ -1,4 +1,4 @@
-use bitstream_reader::{BitRead};
+use bitstream_reader::BitRead;
 use enum_primitive_derive::Primitive;
 
 use crate::{Parse, ParserState, Result, Stream};

src/demo/packet/stop.rs

@@ -1,6 +1,6 @@
 use bitstream_reader::{BitRead, LittleEndian};
 
-use crate::{Stream, ReadResult};
+use crate::{ReadResult, Stream};
 
 #[derive(Debug)]
 pub struct StopPacket;

src/demo/packet/stringtable.rs

@@ -2,18 +2,25 @@ use std::fmt;
 
 use bitstream_reader::{BitRead, LittleEndian};
 
-use crate::{Parse, ParseError, ParserState, ReadResult, Result, Stream};
 use crate::demo::sendprop::SendPropFlag::Exclude;
+use crate::{Parse, ParseError, ParserState, ReadResult, Result, Stream};
+
+#[derive(BitRead, Clone, Copy, Debug)]
+pub struct FixedUserdataSize {
+    #[size = 12]
+    pub size: u16,
+    #[size = 4]
+    pub bits: u8,
+}
 
 #[derive(Debug)]
 pub struct StringTable {
-    name: String,
-    entries: Vec<StringTableEntry>,
-    max_entries: u32,
-    fixed_userdata_size: Option<u32>,
-    fixed_userdata_size_bits: Option<u32>,
-    client_entries: Option<Vec<StringTableEntry>>,
-    compressed: bool,
+    pub name: String,
+    pub entries: Vec<StringTableEntry>,
+    pub max_entries: u16,
+    pub fixed_userdata_size: Option<FixedUserdataSize>,
+    pub client_entries: Option<Vec<StringTableEntry>>,
+    pub compressed: bool,
 }
 
 impl BitRead<LittleEndian> for StringTable {
@@ -34,26 +41,25 @@ impl BitRead<LittleEndian> for StringTable {
             entries,
             max_entries: entry_count,
             fixed_userdata_size: None,
-            fixed_userdata_size_bits: None,
             client_entries,
             compressed: false,
         })
     }
 }
 
-#[derive(BitRead)]
+#[derive(BitRead, Clone)]
 #[endianness = "LittleEndian"]
 pub struct ExtraData {
-    len: u16,
+    pub len: u16,
     #[size = "len * 8"]
-    data: Stream,
+    pub data: Stream,
 }
 
-#[derive(BitRead)]
+#[derive(BitRead, Clone)]
 #[endianness = "LittleEndian"]
 pub struct StringTableEntry {
-    text: String,
-    extra_data: Option<ExtraData>,
+    pub text: String,
+    pub extra_data: Option<ExtraData>,
 }
 
 impl fmt::Debug for StringTableEntry {
@@ -63,8 +69,7 @@ impl fmt::Debug for StringTableEntry {
             Some(extra_data) => write!(
                 f,
                 "StringTableEntry{{ '{}' with {} bits of extra data }}",
-                self.text,
-                extra_data.len
+                self.text, extra_data.len
             ),
         }
     }

src/demo/packet/synctick.rs

@@ -1,4 +1,4 @@
-use bitstream_reader::{BitRead};
+use bitstream_reader::BitRead;
 
 #[derive(BitRead, Debug)]
 pub struct SyncTickPacket {

src/demo/packet/usercmd.rs

@@ -1,6 +1,6 @@
 use bitstream_reader::{BitRead, LittleEndian};
 
-use crate::{Stream, ReadResult};
+use crate::{ReadResult, Stream};
 
 #[derive(Debug)]
 pub struct UserCmdPacket {

src/demo/parser/mod.rs

@@ -1,4 +1,4 @@
-use bitstream_reader::{ReadError, BitRead, LittleEndian};
+use bitstream_reader::{BitRead, LittleEndian, ReadError};
 
 use crate::demo::header::Header;
 use crate::demo::packet::Packet;

src/demo/sendprop.rs

@@ -24,10 +24,7 @@ pub struct SendPropDefinition {
 }
 
 impl SendPropDefinition {
-    pub fn read(
-        stream: &mut Stream,
-        owner_table_name: String,
-    ) -> Result<Self> {
+    pub fn read(stream: &mut Stream, owner_table_name: String) -> Result<Self> {
         let prop_type = SendPropType::read(stream)?;
         let name = stream.read_string(None)?;
         let flags = SendPropFlags::read(stream)?;