handle protocol v23 string tables

src/bin/reencode.rs

@@ -1,16 +1,21 @@
+#![allow(unused_imports)]
+
 use std::env;
 use std::fs;
 
-use bitbuffer::{BitRead, BitWrite, BitWriteStream, LittleEndian};
+use bitbuffer::{BitRead, BitReadBuffer, BitReadStream, BitWrite, BitWriteStream, LittleEndian};
 use main_error::MainError;
+use std::collections::HashMap;
 use steamid_ng::SteamID;
 use tf_demo_parser::demo::data::UserInfo;
 use tf_demo_parser::demo::header::Header;
+use tf_demo_parser::demo::message::stringtable::UpdateStringTableMessage;
 use tf_demo_parser::demo::message::Message;
 use tf_demo_parser::demo::packet::stringtable::{StringTable, StringTableEntry};
 use tf_demo_parser::demo::packet::{Packet, PacketType};
+use tf_demo_parser::demo::parser::gamestateanalyser::UserId;
 use tf_demo_parser::demo::parser::{DemoHandler, Encode, NullHandler, RawPacketStream};
-use tf_demo_parser::{Demo, Parse};
+use tf_demo_parser::{Demo, MessageType, Parse};
 
 const COPY_TYPES: &[PacketType] = &[
     // PacketType::Sigon,
@@ -18,7 +23,7 @@ const COPY_TYPES: &[PacketType] = &[
     // PacketType::SyncTick, // bit perfect
     // PacketType::ConsoleCmd, // bit perfect
     // PacketType::DataTables, // bit perfect
-    // PacketType::StringTables, // bit perfect
+    // PacketType::StringTables, // clone enough
     // PacketType::UserCmd,      // bit perfect
 ];
 
@@ -48,57 +53,45 @@ fn main() -> Result<(), MainError> {
         let mut handler = DemoHandler::parse_all_with_analyser(NullHandler);
 
         let mut packet_start = packets.pos();
-        let mut userinfo_table_id = 0;
 
         while let Some(mut packet) = packets.next(&handler.state_handler)? {
             let packet_end = packets.pos();
             let packet_bits = stream.read_bits(packet_end - packet_start)?;
-            assert_eq!(
-                Packet::parse(&mut packet_bits.clone(), &handler.state_handler)?,
-                packet
-            );
             if COPY_TYPES.contains(&packet.packet_type()) {
                 packet_bits.write(&mut out_stream)?;
             } else {
                 match &mut packet {
                     Packet::Sigon(message_packet) | Packet::Message(message_packet) => {
-                        message_packet.meta.view_angles = Default::default();
+                        // message_packet.meta.view_angles = Default::default();
-                        for message in message_packet.messages.iter_mut() {
+                        let messages = std::mem::take(&mut message_packet.messages);
-                            match message {
+                        let messages = messages
-                                Message::CreateStringTable(table_message) => {
+                            .into_iter()
-                                    if table_message.table.name == "userinfo" {
+                            .filter(|msg| msg.get_message_type() != MessageType::SetView)
-                                        for (_i, entry) in table_message.table.entries.iter_mut() {
+                            .map(|mut msg| {
-                                            mut_string_user_info(entry);
+                                match &mut msg {
-                                        }
+                                    Message::ServerInfo(info) => {
-                                    }
+                                        info.stv = true;
-                                }
-                                Message::UpdateStringTable(table_message) => {
-                                    let table_name = &handler.string_table_names
-                                        [table_message.table_id as usize];
-                                    if table_name == "userinfo" {
-                                        for (_i, entry) in table_message.entries.iter_mut() {
-                                            mut_string_user_info(entry);
                                     }
+                                    _ => {}
+                                };
+                                msg
+                            })
+                            .collect::<Vec<_>>();
+                        message_packet.messages = messages;
                     }
+                    Packet::ConsoleCmd(cmd) => {
+                        println!("{}", cmd.command);
                     }
                     _ => {}
                 }
+
+                if packet.packet_type() != PacketType::ConsoleCmd {
+                    packet
+                        .encode(&mut out_stream, &handler.state_handler)
+                        .unwrap();
                 }
             }
-                    Packet::StringTables(table_packet) => {
+            handler.handle_packet(packet).unwrap();
-                        for table in table_packet.tables.iter_mut() {
-                            if table.name == "userinfo" {
-                                for (_i, entry) in table.entries.iter_mut() {
-                                    mut_string_user_info(entry);
-                                }
-                            }
-                        }
-                    }
-                    _ => {}
-                }
-                packet.encode(&mut out_stream, &handler.state_handler)?;
-            }
-            handler.handle_packet(packet)?;
             packet_start = packet_end;
         }
         assert_eq!(false, packets.incomplete);
@@ -108,19 +101,3 @@ fn main() -> Result<(), MainError> {
 
     Ok(())
 }
-
-fn mut_string_user_info(entry: &mut StringTableEntry) {
-    if let Some(mut user_info) = UserInfo::parse_from_string_table(
-        entry.text.as_deref(),
-        entry.extra_data.as_ref().map(|data| data.data.clone()),
-    )
-    .unwrap()
-    {
-        // dbg!(&user_info);
-        user_info.player_info.name = "[GC]Kimo".into();
-        user_info.player_info.steam_id = "[U:1:32061783]".into();
-        user_info.player_info.friends_id =
-            SteamID::from_steam3("[U:1:32061783]").unwrap().account_id();
-        *entry = user_info.encode_to_string_table().unwrap();
-    }
-}

src/demo/lzss.rs

@@ -0,0 +1,46 @@
+use std::convert::TryInto;
+
+pub fn decompress(input: &[u8], output: &mut Vec<u8>) {
+    decompress_(input, output);
+}
+
+/// inner fn that returns an option so we can use ? for short circuiting return
+fn decompress_(input: &[u8], output: &mut Vec<u8>) -> Option<()> {
+    let target_len = u32::from_le_bytes(input[0..4].try_into().unwrap()) as usize;
+
+    let mut read_pos = 4;
+    let mut read_byte = move || {
+        let byte = *input.get(read_pos)?;
+        read_pos += 1;
+        Some(byte)
+    };
+
+    loop {
+        let mut cmd_byte = read_byte()?;
+
+        for _ in 0..8 {
+            if cmd_byte & 0x01 == 0x01 {
+                let pos = (read_byte()? as usize) << 4;
+                let mixed = read_byte()? as usize;
+                let pos = pos | (mixed >> 4);
+                let count = (mixed & 0x0F) + 1;
+                if count == 1 {
+                    return None;
+                }
+
+                if output.len() + count > target_len {
+                    return None;
+                }
+
+                let start = output.len() - pos - 1;
+                // can't do extend_from_within since it start + count can be larger than output.len
+                for i in 0..count {
+                    output.push(output[start + i]);
+                }
+            } else {
+                output.push(read_byte()?);
+            }
+            cmd_byte >>= 1;
+        }
+    }
+}

src/demo/message/gameevent.rs

@@ -86,7 +86,7 @@ fn test_game_event_roundtrip() {
             entries: vec![],
         },
     ];
-    let mut state = ParserState::new(|_| false, false);
+    let mut state = ParserState::new(24, |_| false, false);
     state.event_definitions = definitions;
 
     crate::test_roundtrip_encode(

src/demo/message/packetentities.rs

@@ -457,7 +457,7 @@ fn test_packet_entitier_message_roundtrip() {
     use crate::demo::packet::datatable::{SendTable, SendTableName, ServerClass, ServerClassName};
     use crate::demo::sendprop::{FloatDefinition, SendPropDefinition, SendPropParseDefinition};
 
-    let mut state = ParserState::new(|_| false, false);
+    let mut state = ParserState::new(24, |_| false, false);
     state.server_classes = vec![
         ServerClass {
             id: ClassId::from(0),

src/demo/message/stringtable.rs

@@ -4,6 +4,7 @@ use bitbuffer::{
 use num_traits::{PrimInt, Unsigned};
 use snap::raw::{decompress_len, Decoder};
 
+use crate::demo::lzss::decompress;
 use crate::demo::packet::stringtable::{
     ExtraData, FixedUserDataSize, StringTable, StringTableEntry,
 };
@@ -33,12 +34,16 @@ impl From<&StringTable<'_>> for StringTableMeta {
 }
 
 impl<'a> Parse<'a> for CreateStringTableMessage<'a> {
-    fn parse(stream: &mut Stream<'a>, _state: &ParserState) -> Result<Self> {
+    fn parse(stream: &mut Stream<'a>, state: &ParserState) -> Result<Self> {
         let name = stream.read()?;
         let max_entries: u16 = stream.read()?;
         let encode_bits = log_base2(max_entries);
         let entity_count: u16 = stream.read_sized(encode_bits as usize + 1)?;
-        let length = read_var_int(stream)?;
+        let length = if state.protocol_version > 23 {
+            read_var_int(stream)?
+        } else {
+            stream.read_sized(20)?
+        };
 
         let fixed_userdata_size = stream.read()?;
 
@@ -64,10 +69,8 @@ impl<'a> Parse<'a> for CreateStringTableMessage<'a> {
 
             let magic = table_data.read_string(Some(4))?;
 
-            if magic != "SNAP" {
+            match magic.as_ref() {
-                return Err(ParseError::UnexpectedCompressionType(magic.into_owned()));
+                "SNAP" => {
-            }
-
                     let compressed_data = table_data.read_bytes(compressed_size as usize - 4)?;
 
                     let mut decoder = Decoder::new();
@@ -89,6 +92,26 @@ impl<'a> Parse<'a> for CreateStringTableMessage<'a> {
                     let buffer = BitReadBuffer::new_owned(decompressed_data, LittleEndian);
                     table_data = BitReadStream::new(buffer);
                 }
+                "LZSS" => {
+                    let compressed_data = table_data.read_bytes(compressed_size as usize - 4)?;
+                    let mut decompressed_data = Vec::with_capacity(decompressed_size as usize);
+                    decompress(&compressed_data, &mut decompressed_data);
+
+                    if decompressed_data.len() != decompressed_size as usize {
+                        return Err(ParseError::UnexpectedDecompressedSize {
+                            expected: decompressed_size,
+                            size: decompressed_data.len() as u32,
+                        });
+                    }
+
+                    let buffer = BitReadBuffer::new_owned(decompressed_data, LittleEndian);
+                    table_data = BitReadStream::new(buffer);
+                }
+                _ => {
+                    return Err(ParseError::UnexpectedCompressionType(magic.into_owned()));
+                }
+            }
+        }
 
         let table_meta = StringTableMeta {
             max_entries,
@@ -171,7 +194,7 @@ impl Encode for CreateStringTableMessage<'_> {
 
 #[test]
 fn test_create_string_table_roundtrip() {
-    let state = ParserState::new(|_| false, false);
+    let state = ParserState::new(24, |_| false, false);
     crate::test_roundtrip_encode(
         CreateStringTableMessage {
             table: StringTable {
@@ -270,7 +293,7 @@ impl Encode for UpdateStringTableMessage<'_> {
 
 #[test]
 fn test_update_string_table_roundtrip() {
-    let mut state = ParserState::new(|_| false, false);
+    let mut state = ParserState::new(24, |_| false, false);
     state.string_tables = vec![StringTableMeta {
         max_entries: 16,
         fixed_userdata_size: None,

src/demo/mod.rs

@@ -5,6 +5,7 @@ pub mod data;
 pub mod gameevent_gen;
 pub mod gamevent;
 pub mod header;
+pub mod lzss;
 pub mod message;
 pub mod packet;
 pub mod parser;

src/demo/packet/datatable.rs

@@ -163,7 +163,7 @@ impl BitWrite<LittleEndian> for ParseSendTable {
 fn test_parse_send_table_roundtrip() {
     use crate::demo::sendprop::SendPropFlags;
 
-    let state = ParserState::new(|_| false, false);
+    let state = ParserState::new(24, |_| false, false);
     crate::test_roundtrip_encode(
         ParseSendTable {
             name: "foo".into(),
@@ -376,7 +376,7 @@ impl BitWrite<LittleEndian> for DataTablePacket {
 fn test_data_table_packet_roundtrip() {
     use crate::demo::sendprop::SendPropFlags;
 
-    let state = ParserState::new(|_| false, false);
+    let state = ParserState::new(24, |_| false, false);
     crate::test_roundtrip_encode(
         DataTablePacket {
             tick: 123,

src/demo/packet/message.rs

@@ -22,7 +22,7 @@ pub struct MessagePacket<'a> {
     pub meta: LazyBitRead<'a, MessagePacketMeta, LittleEndian>,
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Default)]
 pub struct ViewAngles {
     pub origin: (Vector, Vector),
     pub angles: (Vector, Vector),

src/demo/packet/stringtable.rs

@@ -248,7 +248,7 @@ impl Encode for StringTablePacket<'_> {
 
 #[test]
 fn test_string_table_packet_roundtrip() {
-    let state = ParserState::new(|_| false, false);
+    let state = ParserState::new(24, |_| false, false);
     crate::test_roundtrip_encode(
         StringTablePacket {
             tick: 1,

src/demo/parser/handler.rs

@@ -5,6 +5,7 @@ use crate::demo::packet::Packet;
 use crate::demo::parser::analyser::Analyser;
 use crate::Result;
 
+use crate::demo::header::Header;
 use crate::ParserState;
 use std::borrow::Cow;
 
@@ -62,7 +63,7 @@ impl<'a> Default for DemoHandler<'a, Analyser> {
 
 impl<'a, T: MessageHandler> DemoHandler<'a, T> {
     pub fn with_analyser(analyser: T) -> Self {
-        let state_handler = ParserState::new(T::does_handle, false);
+        let state_handler = ParserState::new(24, T::does_handle, false);
 
         DemoHandler {
             tick: 0,
@@ -72,7 +73,7 @@ impl<'a, T: MessageHandler> DemoHandler<'a, T> {
         }
     }
     pub fn parse_all_with_analyser(analyser: T) -> Self {
-        let state_handler = ParserState::new(T::does_handle, true);
+        let state_handler = ParserState::new(24, T::does_handle, true);
 
         DemoHandler {
             tick: 0,
@@ -82,6 +83,10 @@ impl<'a, T: MessageHandler> DemoHandler<'a, T> {
         }
     }
 
+    pub fn handle_header(&mut self, header: &Header) {
+        self.state_handler.protocol_version = header.protocol;
+    }
+
     pub fn handle_packet(&mut self, packet: Packet<'a>) -> Result<()> {
         match packet {
             Packet::DataTables(packet) => {

src/demo/parser/mod.rs

@@ -97,6 +97,7 @@ impl<'a, A: MessageHandler> DemoParser<'a, A> {
     /// while allowing to see the intermediate states
     pub fn ticker(mut self) -> Result<(Header, DemoTicker<'a, A>)> {
         let header = Header::read(&mut self.stream)?;
+        self.handler.handle_header(&header);
         let ticker = DemoTicker {
             handler: self.handler,
             packets: RawPacketStream::new(self.stream),
@@ -135,7 +136,7 @@ impl<'a> RawPacketStream<'a> {
                     Ok(Some(packet))
                 }
                 Ok(packet) => Ok(Some(packet)),
-                Err(ParseError::ReadError(BitError::NotEnoughData { .. })) => {
+                Err(ParseError::ReadError(BitError::NotEnoughData { .. })) if false => {
                     self.ended = true;
                     self.incomplete = true;
                     Ok(None)

src/demo/parser/state.rs

@@ -37,6 +37,7 @@ pub struct ParserState {
     analyser_handles: fn(message_type: MessageType) -> bool,
     handle_entities: bool,
     parse_all: bool,
+    pub protocol_version: u32,
 }
 
 #[derive(Clone)]
@@ -58,7 +59,11 @@ impl StaticBaseline {
 }
 
 impl<'a> ParserState {
-    pub fn new(analyser_handles: fn(message_type: MessageType) -> bool, parse_all: bool) -> Self {
+    pub fn new(
+        protocol_version: u32,
+        analyser_handles: fn(message_type: MessageType) -> bool,
+        parse_all: bool,
+    ) -> Self {
         ParserState {
             static_baselines: HashMap::with_hasher(NullHasherBuilder),
             parsed_static_baselines: RefCell::new(HashMap::with_hasher(NullHasherBuilder)),
@@ -75,6 +80,7 @@ impl<'a> ParserState {
             analyser_handles,
             handle_entities: analyser_handles(MessageType::PacketEntities) || parse_all,
             parse_all,
+            protocol_version,
         }
     }
 

tests/reencode.rs

@@ -101,14 +101,20 @@ fn re_encode_test(input_file: &str) {
                                     .props
                                     .iter()
                                     .map(|prop| {
-                                        (prop_names.get(&prop.index).unwrap(), prop.value.clone())
+                                        (
+                                            prop_names.get(&prop.identifier).unwrap(),
+                                            prop.value.clone(),
+                                        )
                                     })
                                     .collect::<Vec<_>>();
                                 let re_props = re_ent
                                     .props
                                     .iter()
                                     .map(|prop| {
-                                        (prop_names.get(&prop.index).unwrap(), prop.value.clone())
+                                        (
+                                            prop_names.get(&prop.identifier).unwrap(),
+                                            prop.value.clone(),
+                                        )
                                     })
                                     .collect::<Vec<_>>();
                                 pretty_assertions::assert_eq!(props, re_props);