string table fixes

Cargo.lock

@@ -22,6 +22,11 @@ dependencies = [
  "syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "byteorder"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "enum-primitive-derive"
 version = "0.1.2"
@@ -47,6 +52,11 @@ dependencies = [
  "syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "num-traits"
 version = "0.1.43"
@@ -81,6 +91,15 @@ dependencies = [
  "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "snap"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "syn"
 version = "0.11.11"
@@ -119,6 +138,7 @@ dependencies = [
  "enumflags2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "enumflags2_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -133,14 +153,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [metadata]
 "checksum arraydeque 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e300327073b806ffc81fccb228b2d4131ac7ef1b1a015f7b0c399c7f886cacc6"
+"checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb"
 "checksum enum-primitive-derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2b90e520ec62c1864c8c78d637acbfe8baf5f63240f2fb8165b8325c07812dd"
 "checksum enumflags2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "801303a673e02d2110a196d70e4a807f0ff0a68ce1060eebc1e6a0d598db65e8"
 "checksum enumflags2_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e015b3dfedc096cb55cdc5d022d6b4e6b94547212fb94ad2d9ece20bcd88fe3"
+"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14"
 "checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
 "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1"
 "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915"
 "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
 "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1"
+"checksum snap 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "95d697d63d44ad8b78b8d235bf85b34022a78af292c8918527c5f0cffdde7f43"
 "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
 "checksum syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)" = "f92e629aa1d9c827b2bb8297046c1ccffc57c99b947a680d3ccff1f136a3bee9"
 "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"

Cargo.toml

@@ -18,3 +18,4 @@ num-traits = "0.2"
 enumflags2 = "0.5"
 enumflags2_derive = "0.5"
 arraydeque = "0.4"
+snap = "0.2"

src/demo/message/stringtable.rs

@@ -1,10 +1,12 @@
 use std::collections::HashMap;
 
 use arraydeque::{ArrayDeque, Wrapping};
-use bitstream_reader::{BitRead, BitReadSized, BitStream, LittleEndian};
+use bitstream_reader::{BitRead, BitReadSized, BitStream, LittleEndian, BitBuffer};
+use snap::Decoder;
 
-use crate::{Parse, ParseError, ParserState, ReadResult, Stream, Result};
+use crate::{Parse, ParseError, ParserState, ReadResult, Result, Stream};
 use crate::demo::packet::stringtable::{ExtraData, FixedUserdataSize, StringTable, StringTableEntry};
+use num_traits::{PrimInt, Unsigned};
 
 #[derive(Debug)]
 pub struct CreateStringTableMessage {
@@ -26,11 +28,11 @@ impl From<&StringTable> for StringTableMeta {
     }
 }
 
-impl BitRead<LittleEndian> for CreateStringTableMessage {
+impl Parse for CreateStringTableMessage {
-    fn read(stream: &mut Stream) -> ReadResult<Self> {
+    fn parse(stream: &mut Stream, state: &ParserState) -> Result<Self> {
         let name = stream.read()?;
         let max_entries: u16 = stream.read()?;
-        let encode_bits = 16 - max_entries.leading_zeros();
+        let encode_bits = log_base2(max_entries);
         let entity_count: u16 = stream.read_sized(encode_bits as usize + 1)?;
         let bit_count = read_var_int(stream)?;
 
@@ -38,10 +40,32 @@ impl BitRead<LittleEndian> for CreateStringTableMessage {
 
         let compressed = stream.read()?;
 
-        let table_data = stream.read_bits(bit_count as usize)?;
+        let mut table_data = stream.read_bits(bit_count as usize)?;
 
         if compressed {
-            unimplemented!("TODO: SNAP decoding");
+            let decompressed_size: u32 = table_data.read()?;
+            let compressed_size: u32 = table_data.read()?;
+
+            let magic = table_data.read_string(Some(4))?;
+
+            if magic != "SNAP" {
+                return Err(ParseError::UnexpectedCompressionType(magic));
+            }
+
+            let compressed_data = table_data.read_bytes(compressed_size as usize - 4)?;
+
+            let mut decoder = Decoder::new();
+            let decompressed_data = decoder.decompress_vec(&compressed_data).map_err(ParseError::from)?;
+
+            if decompressed_data.len() != decompressed_size as usize {
+                return Err(ParseError::UnexpectedDecompressedSize {
+                    expected: decompressed_size,
+                    size: decompressed_data.len() as u32
+                });
+            }
+
+            let buffer = BitBuffer::new(decompressed_data, LittleEndian);
+            table_data = BitStream::new(buffer);
         }
 
         let table_meta = StringTableMeta {
@@ -49,14 +73,16 @@ impl BitRead<LittleEndian> for CreateStringTableMessage {
             fixed_userdata_size,
         };
 
-        let entries = parse_string_table_entries(stream, &table_meta, entity_count, &Vec::new())?;
+        let entries = parse_string_table_entries(&mut table_data, &table_meta, entity_count, &Vec::new())?;
         let mut entries: Vec<(u16, StringTableEntry)> = entries.into_iter().collect();
 
         // verify that there are no holes in our indexes
         entries.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
-        let last_index = entries.last().map(|(index, _)| *index).unwrap_or(0u16) as usize;
+        if entries.len() > 0 {
-        if last_index != entries.len() {
+            let last_index = entries.last().map(|(index, _)| *index).unwrap_or(0u16) as usize;
-            panic!("there should be no holes when reading CreateStringTable message");
+            if last_index != entries.len() - 1 {
+                panic!("there should be no holes when reading CreateStringTable message");
+            }
         }
         let table_entries = entries.into_iter().map(|(_, entry)| entry).collect();
         let table = StringTable {
@@ -106,11 +132,11 @@ fn parse_string_table_entries(
     entry_count: u16,
     existing_entries: &Vec<StringTableEntry>,
 ) -> ReadResult<HashMap<u16, StringTableEntry>> {
-    let entry_bits = 16 - table_meta.max_entries.leading_zeros();
+    let entry_bits = log_base2(table_meta.max_entries);
     let mut entries = HashMap::with_capacity(entry_count as usize);
 
     let mut last_entry: i16 = -1;
-    let mut history: ArrayDeque<[String; 32], Wrapping> = ArrayDeque::new();
+    let mut history: Vec<String> = Vec::new();
 
     for i in 0..entry_count {
         let index = if stream.read()? {
@@ -119,6 +145,8 @@ fn parse_string_table_entries(
             stream.read_sized(entry_bits as usize)?
         };
 
+        last_entry = index as i16;
+
         let value = if stream.read()? { // set value
             if stream.read()? { // reuse from history
                 let index: u16 = stream.read_sized(5)?;
@@ -143,7 +171,7 @@ fn parse_string_table_entries(
                 Some(size) => stream.read_bits(size.bits as usize)?,
                 None => {
                     let bytes: u16 = stream.read_sized(14)?;
-                    stream.read_bits(bytes as usize)?
+                    stream.read_bits(bytes as usize * 8)?
                 }
             })
         } else {
@@ -171,9 +199,11 @@ fn parse_string_table_entries(
         // `entries` always outlives `history` without reallocation
         let text = entry.text.clone();
         entries.insert(index, entry);
-        unsafe {
+        // not 100% sure we should be pushing front here, and not appending
-            // not 100% sure we should be pushing front here, and not appending
+        history.push(text);
-            history.push_front(text);
+
+        if history.len() > 32 {
+            history.remove(0);
         }
     }
 
@@ -192,3 +222,7 @@ pub fn read_var_int(stream: &mut Stream) -> ReadResult<u32> {
     }
     Ok(result)
 }
+
+pub fn log_base2<T: PrimInt + Unsigned>(num: T) -> u32 {
+    (std::mem::size_of::<T>() as u32 * 8 - 1) - num.leading_zeros()
+}

src/demo/packet/message.rs

@@ -16,8 +16,8 @@ pub struct MessagePacket {
 
 impl Parse for MessagePacket {
     fn parse(stream: &mut Stream, state: &ParserState) -> Result<Self> {
-        let tick = stream.read_int(32)?;
+        let tick = stream.read()?;
-        let flags = stream.read_int(32)?;
+        let flags = stream.read()?;
 
         let view_origin_1 = Vector::parse(stream, state)?;
         let view_angle_1 = Vector::parse(stream, state)?;
@@ -26,16 +26,14 @@ impl Parse for MessagePacket {
         let view_angles = (Vector::parse(stream, state)?, view_angle_1);
         let local_view_angles = (Vector::parse(stream, state)?, local_view_angle_1);
 
-        let sequence_in = stream.read_int(32)?;
+        let sequence_in = stream.read()?;
-        let sequence_out = stream.read_int(32)?;
+        let sequence_out = stream.read()?;
-        let length: usize = stream.read_int(32)?;
+        let length: u32 = stream.read()?;
-        let mut packet_data = stream.read_bits(length * 8)?;
+        let mut packet_data = stream.read_bits(length as usize * 8)?;
 
-        let mut messages = Vec::new();
+        let mut messages: Vec<Message> = Vec::new();
-        dbg!(&packet_data);
         while packet_data.bits_left() > 6 {
-            let message = Message::parse(stream, state)?;
+            let message = Message::parse(&mut packet_data, state)?;
-            dbg!(&message);
             messages.push(message);
         }
 

src/demo/packet/stringtable.rs

@@ -75,10 +75,10 @@ pub struct StringTableEntry {
 impl fmt::Debug for StringTableEntry {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match &self.extra_data {
-            None => write!(f, "Table Entry: '{}'", self.text),
+            None => write!(f, "StringTableEntry {{ text: {} }}", self.text),
             Some(extra_data) => write!(
                 f,
-                "StringTableEntry{{ '{}' with {} bits of extra data }}",
+                "StringTableEntry{{ text: {} extra_data: {} bits }}",
                 self.text, extra_data.len
             ),
         }

src/demo/parser/mod.rs

@@ -34,6 +34,17 @@ pub enum ParseError {
         name: String,
         value: GameEventValue,
     },
+    /// Unexpected type of compressed data
+    UnexpectedCompressionType(String),
+    /// Error while decompressing SNAP compressed string table
+    SnapError(snap::Error),
+    /// Unexpected size after decompressing SNAP data
+    UnexpectedDecompressedSize {
+        /// Expected decompressed size
+        expected: u32,
+        /// Actual decompressed size
+        size: u32
+    }
 }
 
 impl From<ReadError> for ParseError {
@@ -42,6 +53,12 @@ impl From<ReadError> for ParseError {
     }
 }
 
+impl From<snap::Error> for ParseError {
+    fn from(err: snap::Error) -> ParseError {
+        ParseError::SnapError(err)
+    }
+}
+
 pub type Result<T> = std::result::Result<T, ParseError>;
 
 pub trait Parse: Sized {