From 7c5c596d178df8891fc4f32d5bf701df8159e4a2 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 31 Aug 2026 10:50:23 +0100 Subject: [PATCH 1/2] Use the demo's protocol version when reencoding DemoHandler::default() starts out at protocol 24 and relies on handle_header to replace that with whatever the demo actually is. Parsing through DemoParser does this for you, but reencode drives the handler itself and never did, so every demo got read as protocol 24. Older protocols put a few fields at different widths, so the message stream desyncs partway in. RawPacketStream reports that as a truncated demo rather than an error, which is why nothing complains about it: reencode_demo on test_data/protocol23.dem writes 1076 bytes, prints a truncation warning and still exits 0. The reencode test missed this because protocol23.dem was never in its list, and because nothing checked the demo had been read to the end. Add a test for that and the same assert to the existing round trip. Output for protocol 24 demos is unchanged, byte for byte. --- src/bin/reencode.rs | 7 +++++++ tests/reencode.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/bin/reencode.rs b/src/bin/reencode.rs index fcdcfc1..eb80475 100644 --- a/src/bin/reencode.rs +++ b/src/bin/reencode.rs @@ -45,8 +45,15 @@ fn main() -> Result<(), MainError> { } header.write(&mut out_stream)?; + // ParserState defaults to protocol 24 (DemoHandler::default -> + // ParserState::new(24, ..)); without this the demo's own protocol is + // never applied and older demos are parsed with the wrong field widths. + // DemoParser::ticker does this for us, but this binary drives the + // DemoHandler directly. let mut handler = DemoHandler::default(); + handler.handle_header(&header); let mut encode_handler = DemoHandler::default(); + encode_handler.handle_header(&header); let mut has_stop = false; let mut last_tick = DemoTick::default(); diff --git a/tests/reencode.rs b/tests/reencode.rs index a94a732..2930856 100644 --- a/tests/reencode.rs +++ b/tests/reencode.rs @@ -44,6 +44,7 @@ fn re_encode_test(input_file: &str) { let mut packets = RawPacketStream::new(stream); let mut handler = DemoHandler::parse_all_with_analyser(NullHandler); + handler.handle_header(&header); { let mut out_stream = BitWriteStream::new(&mut out_buffer, LittleEndian); @@ -150,4 +151,36 @@ fn re_encode_test(input_file: &str) { handler.handle_packet(packet).unwrap(); } + + // a desync shows up as the packet loop quietly stopping early, so the + // round trip above passes while most of the demo is never looked at + assert!(!packets.incomplete, "{} was not read to the end", input_file); +} + +/// A demo recorded on an older protocol must be parsed with that protocol, not +/// the protocol 24 that `ParserState` starts out with. Getting it wrong shifts +/// the width of a handful of fields (`ServerInfo`'s map hash, `PreFetch`), the +/// message stream desyncs, and `RawPacketStream` quietly stops early rather +/// than returning an error - so this checks the whole demo was actually read. +#[cfg(feature = "write")] +#[test] +fn parse_with_header_protocol() { + let file = fs::read("test_data/protocol23.dem").expect("Unable to read file"); + let demo = Demo::new(&file); + let mut stream = demo.get_stream(); + let header = Header::read(&mut stream).unwrap(); + assert_eq!(23, header.protocol); + + let mut packets = RawPacketStream::new(stream); + let mut handler = DemoHandler::parse_all_with_analyser(NullHandler); + handler.handle_header(&header); + + let mut count = 0; + while let Some(packet) = packets.next(&handler.state_handler).unwrap() { + handler.handle_packet(packet).unwrap(); + count += 1; + } + + assert!(!packets.incomplete, "demo was not read to the end"); + assert_eq!(128235, count); } From 56f08564411e41e400fe33575b218a94eac424ec Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 31 Aug 2026 11:38:23 +0100 Subject: [PATCH 2/2] Write protocol dependent lengths when encoding CreateStringTable and TempEntities both read their payload length as a fixed width field on protocol 23 and older and as a var int after that, but the encoders always write the var int. Reencoding an older demo therefore hands the game a length it can't read: string tables come back as "bogus substring index", and any packet carrying temp entities is malformed from there on, taking whatever messages follow it in that packet with it. The threshold and both widths are taken from the read paths a few lines up in these same two files rather than worked out separately, so the two sides can't drift: > 23, with 20 bits for CreateStringTable and 17 for TempEntities. The engine agrees, for what it's worth - netmessages.cpp reads both of these under > PROTOCOL_VERSION_23 at NET_MAX_PAYLOAD_BITS_V23 + 3 and NET_MAX_PAYLOAD_BITS_V23 respectively, and that constant is 17. serverinfo and prefetch already branch on the protocol on both sides, these two were the only ones that didn't. On a protocol 15 demo that was 28909 of 92518 reencoded packets failing to parse back; on test_data/protocol23.dem, 13220 of 43327. Both are zero now, so protocol23.dem can join the reencode round trip test. Protocol 24 output is unchanged, byte for byte. --- src/demo/message/stringtable.rs | 13 ++++++++++--- src/demo/message/tempentities.rs | 11 +++++++++-- tests/reencode.rs | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/demo/message/stringtable.rs b/src/demo/message/stringtable.rs index 7673cd3..ad53dce 100644 --- a/src/demo/message/stringtable.rs +++ b/src/demo/message/stringtable.rs @@ -164,7 +164,7 @@ impl Encode for CreateStringTableMessage<'_> { fn encode( &self, stream: &mut BitWriteStream, - _state: &ParserState, + state: &ParserState, ) -> Result<()> { let table = &self.table; table.name.write(stream)?; @@ -172,7 +172,14 @@ impl Encode for CreateStringTableMessage<'_> { let encode_bits = log_base2(table.max_entries) as usize; (table.entries.len() as u16).write_sized(stream, encode_bits + 1)?; - stream.reserve_int::(40, |stream| { + // threshold and both widths are the ones used by the read side above + let (length_bits, encode_length): (usize, fn(u32) -> u64) = + if state.protocol_version > 23 { + (40, encode_var_int_fixed) + } else { + (20, |length| length as u64) + }; + stream.reserve_int::(length_bits, |stream| { table.fixed_user_data_size.is_some().write(stream)?; if let Some(fixed_size) = table.fixed_user_data_size { fixed_size.write(stream)?; @@ -188,7 +195,7 @@ impl Encode for CreateStringTableMessage<'_> { write_string_table_update(&table.entries, stream, &table_meta)?; let end = stream.bit_len(); - Ok(encode_var_int_fixed((end - start) as u32)) + Ok(encode_length((end - start) as u32)) })?; Ok(()) diff --git a/src/demo/message/tempentities.rs b/src/demo/message/tempentities.rs index d9e66aa..4778770 100644 --- a/src/demo/message/tempentities.rs +++ b/src/demo/message/tempentities.rs @@ -109,7 +109,14 @@ impl Encode for TempEntitiesMessage { }; count.write(stream)?; - stream.reserve_int::(40, |stream| { + // threshold and both widths are the ones used by the read side above + let (length_bits, encode_length): (usize, fn(u32) -> u64) = + if state.protocol_version > 23 { + (40, encode_var_int_fixed) + } else { + (17, |length| length as u64) + }; + stream.reserve_int::(length_bits, |stream| { let start = stream.bit_len(); let mut last_class_id = u16::MAX.into(); @@ -138,7 +145,7 @@ impl Encode for TempEntitiesMessage { PacketEntitiesMessage::write_update(&event.props, stream, send_table, 0u32.into())?; } let end = stream.bit_len(); - Ok(encode_var_int_fixed((end - start) as u32)) + Ok(encode_length((end - start) as u32)) })?; Ok(()) } diff --git a/tests/reencode.rs b/tests/reencode.rs index 2930856..6fa731d 100644 --- a/tests/reencode.rs +++ b/tests/reencode.rs @@ -32,6 +32,7 @@ use tf_demo_parser::{Demo, Parse}; #[test_case("test_data/decal.dem"; "decal.dem")] #[test_case("test_data/saytext2.dem"; "saytext2.dem")] #[test_case("test_data/emptysaytext.dem"; "emptysaytext.dem")] +#[test_case("test_data/protocol23.dem"; "protocol23.dem")] fn re_encode_test(input_file: &str) { let file = fs::read(input_file).expect("Unable to read file"); let demo = Demo::new(&file);