read string tables from packets

package.json

@@ -9,7 +9,8 @@
   "dependencies": {
     "bit-buffer": "icewind1991/bit-buffer#typings",
     "clone": "^2.1.0",
-    "minimist": "1.1.x"
+    "minimist": "1.1.x",
+    "snappyjs": "^0.5.0"
   },
   "devDependencies": {
     "@types/node": "^6.0.52",

src/Data/StringTable.ts

@@ -1,9 +1,13 @@
+import {BitStream} from "bit-buffer";
 export interface StringTable {
 	name: string;
-	entries: StringTableEntry[];
+	entries: StringTableEntry[],
+	maxEntries: number;
+	fixedUserDataSize?: number;
+	fixedUserDataSizeBits?: number;
 }
 
 export interface StringTableEntry {
 	text: string;
-	extraData: string[];
+	extraData?: BitStream;
 }

src/Math.ts

@@ -0,0 +1,7 @@
+export function logBase2(num: number): number {
+	let result = 0;
+	while ((num >>= 1) != 0) {
+		result++;
+	}
+	return result;
+}

src/Parser.ts

@@ -96,7 +96,6 @@ export class Parser extends EventEmitter {
 			return false;
 		}
 		const tick = stream.readInt32();
-		let start, length, buffer;
 
 		let viewOrigin: number[][] = [];
 		let viewAngles: number[][] = [];
@@ -128,8 +127,8 @@ export class Parser extends EventEmitter {
 				return true;
 		}
 
-		length = stream.readInt32();
+		const length = stream.readInt32();
-		buffer = stream.readBitStream(length * 8);
+		const buffer = stream.readBitStream(length * 8);
 		return this.parseMessage(buffer, type, tick, length, match);
 	}
 }

src/Parser/EntityDecoder.ts

@@ -5,22 +5,16 @@ import {SendPropParser} from "./SendPropParser";
 import {readUBitVar} from "./readBitVar";
 
 export function applyEntityUpdate(entity: Entity, stream: BitStream): Entity {
-	let index                    = -1;
+	let index      = -1;
-	const allProps               = entity.sendTable.flattenedProps;
+	const allProps = entity.sendTable.flattenedProps;
 	while ((index = readFieldIndex(stream, index)) != -1) {
-		if (index > 4096) {
+		if (index >= 4096 || index > allProps.length) {
-			throw new Error('prop index out of bounds while applying update for ' + entity.sendTable.name + ' got ' + index);
+			throw new Error('prop index out of bounds while applying update for ' + entity.sendTable.name + ' got ' + index
+				+ ' proptype only has ' + allProps.length + ' properties');
 		}
-		console.log("index: " + index, allProps.length);
 		const propDefinition = allProps[index];
-		// console.log(propDefinition);
 		const existingProp   = entity.getPropByDefinition(propDefinition);
-		let prop;
+		const prop = existingProp ? existingProp : new SendProp(propDefinition);
-		if (existingProp) {
-			prop = existingProp;
-		} else {
-			prop = new SendProp(propDefinition);
-		}
 		prop.value = SendPropParser.decode(propDefinition, stream);
 		console.log(prop);
 

src/Parser/Message/DataTable.ts

@@ -2,6 +2,7 @@ import {SendTable} from '../../Data/SendTable';
 import {SendPropDefinition, SendPropFlag, SendPropType} from '../../Data/SendPropDefinition';
 import {ServerClass} from '../../Data/ServerClass';
 import {Parser} from './Parser';
+import {BitStream} from "bit-buffer";
 
 export class DataTable extends Parser {
 	parse() {

src/Parser/Message/StringTable.ts

@@ -4,6 +4,11 @@ import {RemoteInfo} from "dgram";
 
 export class StringTable extends Parser {
 	parse() {
+		// we get the tables from the packets
+		return [{
+			packetType: 'stringTable',
+			tables:     []
+		}];
 		// https://github.com/StatsHelix/demoinfo/blob/3d28ea917c3d44d987b98bb8f976f1a3fcc19821/DemoInfo/ST/StringTableParser.cs
 		const tableCount = this.stream.readUint8();
 		let tables = {};
@@ -56,7 +61,6 @@ export class StringTable extends Parser {
 				}
 			}
 		}
-		//console.log(tables);
 		return [{
 			packetType: 'stringTable',
 			tables: tables

src/Parser/Packet/ClassInfo.ts

@@ -1,13 +1,6 @@
 import {Packet} from "../../Data/Packet";
 import {BitStream} from 'bit-buffer';
-
+import {logBase2} from '../../Math';
-function logBase2(num: number): number {
-	let result = 0;
-	while ((num >>= 1) != 0) {
-		result++;
-	}
-	return result;
-}
 
 export function ClassInfo(stream: BitStream): Packet { // 10: classInfo
 	const number = stream.readBits(16);

src/Parser/Packet/CreateStringTable.ts

@@ -2,11 +2,68 @@ import {PacketStringTable} from './PacketStringTable';
 
 import {Packet} from "../../Data/Packet";
 import {BitStream} from 'bit-buffer';
+import {logBase2} from "../../Math";
+import {readBitVar, readVarInt} from "../readBitVar";
 
-export function CreateStringTable(stream: BitStream): Packet { // 12: createStringTable
+import {uncompress} from "snappyjs";
-	const tables = PacketStringTable(stream);
+import {StringTable} from "../../Data/StringTable";
+import {parseStringTable} from "../StringTableParser";
+import {Match} from "../../Data/Match";
+
+export function CreateStringTable(stream: BitStream, match: Match): Packet { // 12: createStringTable
+	const tableName   = stream.readASCIIString();
+	const maxEntries  = stream.readUint16();
+	const encodeBits  = logBase2(maxEntries);
+	const entityCount = stream.readBits(encodeBits + 1);
+
+	const bitCount = readVarInt(stream);
+
+	let userDataSize     = 0;
+	let userDataSizeBits = 0;
+
+	// userdata fixed size
+	if (stream.readBoolean()) {
+		userDataSize     = stream.readBits(12);
+		userDataSizeBits = stream.readBits(4);
+	}
+
+	const isCompressed = stream.readBoolean();
+
+	let data = stream.readBitStream(bitCount);
+
+	if (isCompressed) {
+		const decompressedByteSize = data.readUint32();
+		const compressedByteSize   = data.readUint32();
+
+		const magic = data.readASCIIString(4);
+
+		const compressedData = data.readArrayBuffer(compressedByteSize - 4); // 4 magic bytes
+
+		if (magic !== 'SNAP') {
+			throw new Error("Unknown compressed stringtable format");
+		}
+
+		const decompressedData = uncompress(compressedData);
+		if (decompressedData.byteLength !== decompressedByteSize) {
+			throw new Error("Incorrect length of decompressed stringtable");
+		}
+
+		data = new BitStream(decompressedData.buffer);
+	}
+
+	const table: StringTable = {
+		name:                  tableName,
+		entries:               [],
+		maxEntries:            maxEntries,
+		fixedUserDataSize:     userDataSize,
+		fixedUserDataSizeBits: userDataSizeBits
+	};
+
+	parseStringTable(data, table, entityCount, match);
+
+	match.stringTables.push(table);
 	return {
 		packetType: 'createStringTable',
-		table: tables
+		table:      table
 	};
 }

src/Parser/Packet/PacketEntities.ts

@@ -27,7 +27,7 @@ function readPVSType(stream: BitStream): PVS {
 	} else if (hi) {
 		pvs = (low) ? (PVS.LEAVE | PVS.DELETE) : PVS.LEAVE;
 	} else {
-		pvs = -1;
+		throw new Error('Invalid pvs');
 	}
 	return pvs;
 }
@@ -58,9 +58,12 @@ function readEnterPVS(stream: BitStream, entityId: number, match: Match, baseLin
 	} else {
 		const staticBaseLine = match.staticBaseLines[serverClass.id];
 		if (staticBaseLine) {
-			const streamStart = staticBaseLine.index;
+			staticBaseLine.index = 0;
 			applyEntityUpdate(entity, staticBaseLine);
-			staticBaseLine.index = streamStart;
+			if(staticBaseLine.bitsLeft > 7) {
+				console.log(staticBaseLine.length, staticBaseLine.index);
+				throw new Error('Unexpected data left at the end of staticBaseline, ' + stream.bitsLeft + ' bits left');
+			}
 		}
 	}
 	return entity;
@@ -77,7 +80,6 @@ export function PacketEntities(stream: BitStream, match: Match): Packet { //26:
 	// https://github.com/StatsHelix/demoinfo/blob/3d28ea917c3d44d987b98bb8f976f1a3fcc19821/DemoInfo/DP/Handler/PacketEntitesHandler.cs
 	// https://github.com/StatsHelix/demoinfo/blob/3d28ea917c3d44d987b98bb8f976f1a3fcc19821/DemoInfo/DP/Entity.cs
 	// https://github.com/PazerOP/DemoLib/blob/5f9467650f942a4a70f9ec689eadcd3e0a051956/TF2Net/NetMessages/NetPacketEntitiesMessage.cs
-	// todo
 	const maxEntries      = stream.readBits(11);
 	const isDelta         = !!stream.readBits(1);
 	const delta           = (isDelta) ? stream.readInt32() : null;
@@ -88,10 +90,10 @@ export function PacketEntities(stream: BitStream, match: Match): Packet { //26:
 	const end             = stream.index + length;
 	let entityId          = -1;
 
-	stream.index = end;
+	// stream.index = end;
-	return {
+	// return {
-		packetType: 'packetEntities'
+	// 	packetType: 'packetEntities'
-	};
+	// };
 
 	if (updatedBaseLine) {
 		if (baseLine === 0) {
@@ -110,6 +112,7 @@ export function PacketEntities(stream: BitStream, match: Match): Packet { //26:
 		console.log("entity: " + entityId, ", pvs " + PVS[pvs]);
 		if (pvs === PVS.ENTER) {
 			const entity = readEnterPVS(stream, entityId, match, baseLine);
+			console.log('got entity');
 			applyEntityUpdate(entity, stream);
 			match.entities[entityId] = entity;
 
@@ -119,7 +122,6 @@ export function PacketEntities(stream: BitStream, match: Match): Packet { //26:
 				match.instanceBaselines[baseLine][entityId] = newBaseLine;
 			}
 			entity.inPVS = true;
-			// stream.readBits(1);
 		} else if (pvs === PVS.PRESERVE) {
 			const entity = match.entities[entityId];
 			if (entity) {

src/Parser/Packet/TempEntities.ts

@@ -15,7 +15,9 @@ export function TempEntities(stream: BitStream, match: Match): Packet { // 10: c
 		const delay = (stream.readBoolean()) ? stream.readUint8() / 100 : 0; //unused it seems
 		if (stream.readBoolean()) {
 			const classId     = stream.readBits(match.classBits);
-			const serverClass = match.serverClasses[classId - 1]; //no clue why the -1 but it works
+			const serverClass = match.serverClasses[classId - 1];	// no clue why the -1 but it works
+																	// maybe because world (id=0) can never be tem
+																	// but it's not like the -1 saves any space
 			const sendTable   = match.getSendTable(serverClass.dataTable);
 			entity            = new Entity(serverClass, sendTable, 0, 0);
 			applyEntityUpdate(entity, stream);

src/Parser/Packet/UpdateStringTable.ts

@@ -1,11 +1,28 @@
 import {PacketStringTable} from './PacketStringTable';
 import {Packet} from "../../Data/Packet";
 import {BitStream} from 'bit-buffer';
+import {Match} from "../../Data/Match";
+import {parseStringTable} from "../StringTableParser";
+
+export function UpdateStringTable(stream: BitStream, match: Match): Packet { // 12: updateStringTable
+	const tableId = stream.readBits(5);
+
+	const multipleChanged = stream.readBoolean();
+	const changedEntries  = (multipleChanged) ? stream.readBits(16) : 1;
+
+	const bitCount = stream.readBits(20);
+	const data     = stream.readBitStream(bitCount);
+
+	if (!match.stringTables[tableId]) {
+		throw new Error('Table not found for update');
+	}
+
+	const table = match.stringTables[tableId];
+	console.log('update table ' + table.name);
+	parseStringTable(data, table, changedEntries, match);
 
-export function UpdateStringTable(stream: BitStream): Packet { // 12: updateStringTable
-	const tables = PacketStringTable(stream);
 	return {
 		packetType: 'updateStringTable',
-		table: tables
+		table:      table
 	};
 }

src/Parser/StringTableParser.ts

@@ -0,0 +1,89 @@
+import {BitStream} from 'bit-buffer';
+import {StringTable, StringTableEntry} from "../Data/StringTable";
+import {logBase2} from "../Math";
+import {Match} from "../Data/Match";
+
+export function parseStringTable(stream: BitStream, table: StringTable, entries: number, match: Match) {
+	const entryBits = logBase2(table.maxEntries);
+	let lastEntry   = -1;
+
+	const history: StringTableEntry[] = [];
+
+	for (let i = 0; i < entries; i++) {
+		let entryIndex = lastEntry + 1;
+
+		if (!stream.readBoolean()) {
+			entryIndex = stream.readBits(entryBits);
+		}
+
+		lastEntry = entryIndex;
+
+		if (entryIndex < 0 || entryIndex > table.maxEntries) {
+			throw new Error("Invalid string index for stringtable");
+		}
+
+		let value;
+
+		if (stream.readBoolean()) {
+			const subStringCheck = stream.readBoolean();
+
+			if (subStringCheck) {
+				const index       = stream.readBits(5);
+				const bytesToCopy = stream.readBits(5);
+
+				const restOfString = stream.readASCIIString();
+
+				value = history[index].text.substr(0, bytesToCopy) + restOfString;
+			} else {
+				value = stream.readASCIIString();
+			}
+		}
+
+		let userData: BitStream|undefined;
+
+		if (stream.readBoolean()) {
+			if (table.fixedUserDataSize && table.fixedUserDataSizeBits) {
+				userData = stream.readBitStream(table.fixedUserDataSizeBits);
+			} else {
+				const userDataBytes = stream.readBits(14);
+				userData            = stream.readBitStream(userDataBytes * 8);
+			}
+		}
+
+		if (table.entries[entryIndex]) {
+			const existingEntry = table.entries[entryIndex];
+			if (userData) {
+				existingEntry.extraData = userData;
+			}
+			if (table.name === 'instancebaseline') {
+				console.log('updating instancebaseline');
+				if (userData) {
+					match.staticBaseLines[parseInt(existingEntry.text, 10)] = userData;
+				} else {
+					throw new Error('Missing baseline');
+				}
+			}
+			history.push(existingEntry);
+
+			if (value) {
+				existingEntry.text = value;
+			}
+		} else {
+			if (table.name === 'instancebaseline') {
+				if (userData) {
+					match.staticBaseLines[parseInt(value, 10)] = userData;
+				} else {
+					throw new Error('Missing baseline');
+				}
+			}
+			table.entries[entryIndex] = {
+				text:      value,
+				extraData: userData
+			};
+			history.push(table.entries[entryIndex]);
+		}
+		if (history.length > 32) {
+			history.shift();
+		}
+	}
+}

src/Parser/readBitVar.ts

@@ -1,6 +1,7 @@
 import {BitStream} from "bit-buffer";
 export function readBitVar(stream: BitStream, signed?: boolean): number {
-	switch (stream.readBits(2)) {
+	const type = stream.readBits(2);
+	switch (type) {
 		case 0:
 			return stream.readBits(4, signed);
 		case 1:
@@ -14,3 +15,18 @@ export function readBitVar(stream: BitStream, signed?: boolean): number {
 }
 
 export const readUBitVar = readBitVar;
+
+
+export function readVarInt(stream: BitStream) {
+	let result = 0;
+	for (let i = 0; i < 35; i += 7) {
+		const byte = stream.readBits(8);
+		result |= ((byte & 0x7F) << i);
+
+		if ((byte >> 7) === 0) {
+			break;
+		}
+	}
+
+	return result;
+}

typings/modules/snappyjs/index.d.ts

@@ -0,0 +1,5 @@
+declare module 'snappyjs' {
+	export function uncompress(input: Uint8Array): Uint8Array;
+
+	export function compress(input: Uint8Array): Uint8Array;
+}