fix encoding string table entries

src/Parser/StringTableParser.ts

@@ -61,7 +61,7 @@ export function parseStringTableEntries(
 				existingEntry.extraData = userData;
 			}
 
-			if (value) {
+			if (typeof value !== 'undefined') {
 				existingEntry.text = value;
 			}
 			entries[entryIndex] = existingEntry;
@@ -97,9 +97,11 @@ export function guessStringTableEntryLength(table: StringTable, entries: StringT
 
 export function encodeStringTableEntries(stream: BitStream, table: StringTable, entries: StringTableEntry[], oldEntries: StringTableEntry[] = []) {
 	const entryBits = logBase2(table.maxEntries);
-	const lastIndex = -1;
+	let lastIndex = -1;
+	const history: StringTableEntry[] = [];
 	for (let i = 0; i < entries.length; i++) {
 		if (entries[i]) {
+
 			const entry = entries[i];
 			if (i !== (lastIndex + 1)) {
 				stream.writeBoolean(false);
@@ -107,17 +109,26 @@ export function encodeStringTableEntries(stream: BitStream, table: StringTable,
 			} else {
 				stream.writeBoolean(true);
 			}
+			lastIndex = i;
 
 			if (typeof entry.text !== 'undefined' && !(oldEntries[i] && entry.text === oldEntries[i].text)) {
 				stream.writeBoolean(true);
-				// we don't encode substring optimizations
-				stream.writeBoolean(false);
 
-				stream.writeASCIIString(entry.text);
+				const {index, count} = getBestPreviousString(history, entry.text);
+				if (index !== -1) {
+					stream.writeBoolean(true);
+					stream.writeBits(index, 5);
+					stream.writeBits(count, 5);
+					stream.writeASCIIString(entry.text.substr(count));
+				} else {
+					stream.writeBoolean(false);
+					stream.writeASCIIString(entry.text);
+				}
 			} else {
 				stream.writeBoolean(false);
 			}
 
+
 			if (entry.extraData) {
 				stream.writeBoolean(true);
 
@@ -133,6 +144,41 @@ export function encodeStringTableEntries(stream: BitStream, table: StringTable,
 			} else {
 				stream.writeBoolean(false);
 			}
+
+
+			history.push(entry);
+			if (history.length > 32) {
+				history.shift();
+			}
 		}
 	}
 }
+
+function getBestPreviousString(history: StringTableEntry[], newString: string): {index: number, count: number} {
+	let bestIndex = -1;
+	let bestCount = 0;
+	for (let i = 0; i < history.length; i++) {
+		const prev = history[i].text;
+		const similar = countSimilarCharacters(prev, newString);
+		if (similar >= 3 && similar > bestCount) {
+			bestCount = similar;
+			bestIndex = i;
+		}
+	}
+	return {
+		index: bestIndex,
+		count: bestCount
+	};
+}
+
+const maxSimLength = 1 << 5;
+
+function countSimilarCharacters(a: string, b: string) {
+	const length = Math.min(a.length, b.length, maxSimLength);
+	for (let i = 0; i < length; i++) {
+		if (a[i] !== b[i]) {
+			return i;
+		}
+	}
+	return Math.min(length, maxSimLength - 1);
+}

src/tests/unit/Parser/Packet/CreateStringTableTest.ts

@@ -60,7 +60,8 @@ const examplePacket = {
 		entries: [{text: 'maps\\pl_badwater_pro_v9.bsp', extraData: undefined}],
 		maxEntries: 8192,
 		fixedUserDataSize: 0,
-		fixedUserDataSizeBits: 0
+		fixedUserDataSizeBits: 0,
+		compressed: false
 	}
 };
 
@@ -76,7 +77,8 @@ const examplePacket2 = {
 		],
 		maxEntries: 8192,
 		fixedUserDataSize: 0,
-		fixedUserDataSizeBits: 0
+		fixedUserDataSizeBits: 0,
+		compressed: false
 	}
 };
 
@@ -87,10 +89,10 @@ suite('CreateStringTable', () => {
 
 	test('Encode createStringTable', () => {
 		assertEncoder(ParseCreateStringTable, EncodeCreateStringTable, examplePacket, 388);
-		assertEncoder(ParseCreateStringTable, EncodeCreateStringTable, examplePacket2, 615);
+		assertEncoder(ParseCreateStringTable, EncodeCreateStringTable, examplePacket2, 562);
 	});
 
-	test('Re-encode classInfo', () => {
+	test('Re-encode createStringTable', () => {
 		assertReEncode(ParseCreateStringTable, EncodeCreateStringTable, getStream(exampleData));
 	});
 });

src/tests/unit/Parser/Packet/PacketTest.ts

@@ -48,7 +48,14 @@ export function assertReEncode(parser: Parser, encoder: Encoder, stream: BitStre
 	encoder(result, encodeStream);
 	assert.equal(encodeStream.index, length, 'Unexpected number of bits used for encoding');
 	encodeStream.index = 0;
-	assert.deepEqual(encodeStream.readArrayBuffer(byteLength), stream.readArrayBuffer(byteLength));
+	const encodeData = encodeStream.readArrayBuffer(byteLength);
+	const originalData = stream.readArrayBuffer(byteLength);
+	for (let i = 0; i < byteLength; i++) {
+		if (originalData[i] !== encodeData[i]) {
+			assert.fail(`Data differs at byte ${i} out of ${byteLength}: ${originalData[i].toString(2)} !== ${encodeData[i].toString(2)}`);
+		}
+	}
+	assert.deepEqual(encodeData, originalData);
 	if (length - 8 * byteLength > 0) {
 		assert.deepEqual(encodeStream.readBits(length - 8 * byteLength), stream.readBits(length - 8 * byteLength));
 	}

src/tests/unit/Parser/Packet/UpdateStringTableTest.ts

@@ -13,7 +13,8 @@ function getExistingParserState() {
 		entries: [],
 		maxEntries: 2048,
 		fixedUserDataSize: 1,
-		fixedUserDataSizeBits: 1
+		fixedUserDataSizeBits: 1,
+		compressed: false,
 	};
 	existingTable.entries[70] = {text: 'maps\\pl_badwater_pro_v9.bsp'};
 	const state = createParserState();

src/tests/unit/Parser/StringTableParserTest.ts

@@ -0,0 +1,40 @@
+import {BitStream} from 'bit-buffer';
+import {assertEncoder, assertParser, assertReEncode} from './Packet/PacketTest';
+import {readFileSync} from 'fs';
+import {encodeStringTableEntries, parseStringTableEntries} from '../../../Parser/StringTableParser';
+import {StringTableEntry} from '../../../Data/StringTable';
+
+const baseTable = {
+	name: 'modelprecache',
+	entries: [],
+	maxEntries: 4096,
+	fixedUserDataSize: 1,
+	fixedUserDataSizeBits: 2,
+	compressed: true
+};
+
+const data = readFileSync(__dirname + '/../../data/stringTableEntries.bin');
+
+function ParseUpdate(stream: BitStream) {
+	//981
+	return parseStringTableEntries(stream, baseTable, 981);
+}
+
+function EncodeUpdate(entries: StringTableEntry[], stream: BitStream) {
+	return encodeStringTableEntries(stream, baseTable, entries);
+}
+
+suite('string table parser', () => {
+	// test('Parse string table entries', () => {
+	// 	assertParser(ParseUpdate, getStream(exampleData), examplePacket, 41);
+	// });
+
+	test('Encode string table entries', () => {
+		const expected = ParseUpdate(new BitStream(data));
+		assertEncoder(ParseUpdate, EncodeUpdate, expected);
+	});
+
+	test('Re-encode string table entries', () => {
+		assertReEncode(ParseUpdate, EncodeUpdate, new BitStream(data));
+	});
+});