1
0
Fork 0
mirror of https://github.com/demostf/demo.js synced 2026-06-04 00:54:14 +02:00

encoder wip

This commit is contained in:
Robin Appelman 2017-10-24 16:59:33 +02:00
commit a5a1642782
13 changed files with 876 additions and 35 deletions

View file

@ -28,9 +28,13 @@ export class ParserState {
public instanceBaselines: [Map<EntityId, SendProp[]>, Map<EntityId, SendProp[]>] = [new Map(), new Map()];
public skippedPackets: PacketTypeId[] = [];
public userInfoEntries: Map<string, BitStream> = new Map();
public tick: number = 0;
public handlePacket(packet: Packet) {
switch (packet.packetType) {
case 'netTick':
this.tick = packet.tick;
break;
case 'serverInfo':
this.version = packet.version;
break;

View file

@ -4,13 +4,8 @@ import {PacketTypeId} from './Data/Packet';
import {Parser} from './Parser';
export class Demo {
public static fromNodeBuffer(nodeBuffer) {
const arrayBuffer = new ArrayBuffer(nodeBuffer.length);
const view = new Uint8Array(arrayBuffer);
for (let i = 0; i < nodeBuffer.length; ++i) {
view[i] = nodeBuffer[i];
}
return new Demo(arrayBuffer);
public static fromNodeBuffer(nodeBuffer: Buffer) {
return new Demo(nodeBuffer.buffer as ArrayBuffer);
}
public stream: BitStream;

48
src/Encoder.ts Normal file
View file

@ -0,0 +1,48 @@
import {BitStream} from 'bit-buffer';
import {Header} from './Data/Header';
import {Message, MessageType} from './Data/Message';
import {ParserState} from './Data/ParserState';
import {messageHandlers} from './Parser';
export class Encoder {
public readonly stream: BitStream;
public readonly parserState: ParserState;
constructor(stream: BitStream) {
this.stream = stream;
this.parserState = new ParserState();
}
public encodeHeader(header: Header) {
this.stream.writeASCIIString(header.type, 8);
this.stream.writeUint32(header.version);
this.stream.writeUint32(header.protocol);
this.stream.writeASCIIString(header.server, 260);
this.stream.writeASCIIString(header.nick, 260);
this.stream.writeASCIIString(header.map, 260);
this.stream.writeASCIIString(header.game, 260);
this.stream.writeFloat32(header.duration);
this.stream.writeUint32(header.ticks);
this.stream.writeUint32(header.frames);
this.stream.writeUint32(header.sigon);
}
public writeMessage(message: Message) {
this.stream.writeUint8(message.type);
const handler = messageHandlers.get(message.type);
if (!handler) {
throw new Error(`No handler for message of type ${MessageType[message.type]}`);
}
handler.encodeMessage(message, this.stream, this.parserState);
this.handleMessage(message);
}
protected handleMessage(message: Message) {
this.parserState.handleMessage(message);
if (message.type === MessageType.Packet) {
for (const packet of message.packets) {
this.parserState.handlePacket(packet);
}
}
}
}

View file

@ -6,18 +6,20 @@ import {ParserState} from './Data/ParserState';
import {ConsoleCmdHandler} from './Parser/Message/ConsoleCmd';
import {DataTableHandler} from './Parser/Message/DataTable';
import {PacketMessageHandler} from './Parser/Message/Packet';
import {StopHandler} from './Parser/Message/Stop';
import {StringTableHandler} from './Parser/Message/StringTable';
import {SyncTickHandler} from './Parser/Message/SyncTick';
import {UserCmdHandler} from './Parser/Message/UserCmd';
const messageHandlers: Map<MessageType, MessageHandler<Message>> = new Map<MessageType, MessageHandler<Message>>([
export const messageHandlers: Map<MessageType, MessageHandler<Message>> = new Map<MessageType, MessageHandler<Message>>([
[MessageType.Sigon, PacketMessageHandler],
[MessageType.Packet, PacketMessageHandler],
[MessageType.ConsoleCmd, ConsoleCmdHandler],
[MessageType.UserCmd, UserCmdHandler],
[MessageType.DataTables, DataTableHandler],
[MessageType.StringTables, StringTableHandler],
[MessageType.SyncTick, SyncTickHandler]
[MessageType.SyncTick, SyncTickHandler],
[MessageType.Stop, StopHandler]
]);
export class Parser {
@ -50,22 +52,13 @@ export class Parser {
protected * getMessages(): Iterable<Message> {
while (true) {
const message = this.readMessage(this.stream, this.parserState);
if (message) {
yield message;
} else {
if (message.type === MessageType.Stop) {
return;
}
}
}
protected parseMessage(data: BitStream, type: MessageType, state: ParserState): Message {
const handler = messageHandlers.get(type);
if (!handler) {
throw new Error(`No handler for message of type ${MessageType[type]}`);
}
return handler.parseMessage(data, state);
}
protected parseHeader(stream): Header {
return {
type: stream.readASCIIString(8),
@ -85,22 +78,28 @@ export class Parser {
protected * handleMessage(message: Message): Iterable<Packet> {
this.parserState.handleMessage(message);
if (message.type === MessageType.Packet) {
for (const packet of (message as PacketMessage).packets) {
for (const packet of message.packets) {
this.parserState.handlePacket(packet);
yield packet;
}
}
}
protected readMessage(stream: BitStream, state: ParserState): Message | false {
protected readMessage(stream: BitStream, state: ParserState): Message {
if (stream.bitsLeft < 8) {
return false;
throw new Error('Stream ended without stop packet');
}
const type: MessageType = stream.readUint8();
if (type === MessageType.Stop) {
return false;
if (type === 0) {
return {
type: MessageType.Stop,
rawData: stream.readBitStream(0)
};
}
return this.parseMessage(stream, type, state);
const handler = messageHandlers.get(type);
if (!handler) {
throw new Error(`No handler for message of type ${MessageType[type]}(${type})`);
}
return handler.parseMessage(this.stream, state);
}
}

View file

@ -9,11 +9,13 @@ export function getEntityUpdate(sendTable: SendTable, stream: BitStream): SendPr
let index = -1;
const allProps = sendTable.flattenedProps;
const props: Map<string, SendProp> = new Map();
let lastIndex = -1;
while (stream.readBoolean()) {
lastIndex = index;
index = readFieldIndex(stream, index);
if (index >= 4096 || index > allProps.length) {
throw new Error('prop index out of bounds while applying update for ' + sendTable.name + ' got ' + index
+ ' property only has ' + allProps.length + ' properties');
throw new Error(`prop index out of bounds while applying update for ${sendTable.name}
got ${index} property only has ${allProps.length} properties (lastProp: ${lastIndex})`);
}
const propDefinition = allProps[index];

View file

@ -0,0 +1,14 @@
import {BitStream} from 'bit-buffer';
import {MessageHandler, MessageType, StopMessage, SyncTickMessage} from '../../Data/Message';
export const StopHandler: MessageHandler<StopMessage> = {
parseMessage: (stream: BitStream) => {
return {
type: MessageType.Stop,
rawData: stream.readBitStream(0)
};
},
encodeMessage: (message, stream) => {
// noop
}
};

View file

@ -4,8 +4,6 @@ import {GameEvent, GameEventType} from '../../Data/GameEventTypes';
import {GameEventListPacket} from '../../Data/Packet';
export function ParseGameEventList(stream: BitStream): GameEventListPacket { // 30: gameEventList
const s = stream.index;
// list of game events and parameters
const numEvents = stream.readBits(9);
const length = stream.readBits(20);
@ -28,6 +26,7 @@ export function ParseGameEventList(stream: BitStream): GameEventListPacket { //
entries
});
}
return {
packetType: 'gameEventList',
eventList

View file

@ -133,6 +133,7 @@ export function ParsePacketEntities(
const updatedEntries = stream.readBits(11);
const length = stream.readBits(20);
const updatedBaseLine = stream.readBoolean();
const start = stream.index;
const end = stream.index + length;
let entityId = -1;
@ -166,6 +167,10 @@ export function ParsePacketEntities(
if (!sendTable) {
throw new Error(`Unknown sendTable ${packetEntity.serverClass.dataTable}`);
}
if (entityId === 55) {
console.log(`decode preserve: ${entityId} = ${sendTable.name}, ${receivedEntities.length}/${i} ${stream.index} ${end} tick ${state.tick}`);
console.log(receivedEntities[receivedEntities.length - 1], start, entityId, diff);
}
const updatedProps = getEntityUpdate(sendTable, stream);
packetEntity.applyPropUpdate(updatedProps);
receivedEntities.push(packetEntity);
@ -173,7 +178,7 @@ export function ParsePacketEntities(
const packetEntity = getPacketEntityForExisting(entityId, state, pvs);
receivedEntities.push(packetEntity);
} else {
// throw new Error(`No existing entity to update with id ${entityId}`);
throw new Error(`No existing entity to update with id ${entityId}`);
}
}
@ -233,6 +238,9 @@ export function EncodePacketEntities(packet: PacketEntitiesPacket, stream: BitSt
writeEnterPVS(entity, stream, state, packet.baseLine);
} else if (entity.pvs === PVS.PRESERVE) {
const sendTable = getSendTable(state, entity.serverClass.dataTable);
if (entity.entityIndex === 55) {
console.log(`encode preserve: ${entity.entityIndex} = ${entity.serverClass.dataTable}`);
}
encodeEntityUpdate(entity.props, sendTable, stream);
}
}

View file

@ -52,7 +52,7 @@ export const readUBitVar = readBitVar;
export function readVarInt(stream: BitStream, signed: boolean = false) {
let result = 0;
for (let i = 0; i < 35; i += 7) {
const byte = stream.readBits(8);
const byte = stream.readUint8();
result |= ((byte & 0x7F) << i);
if ((byte >> 7) === 0) {

38
src/Transformer.ts Normal file
View file

@ -0,0 +1,38 @@
import {BitStream} from 'bit-buffer';
import {Parser} from './Parser';
import {Encoder} from './Encoder';
import {Packet} from './Data/Packet';
import {Message, MessageType} from './Data/Message';
export type PacketTransform = (packet: Packet) => Packet;
export type MessageTransform = (message: Message) => Message;
export function nullTransform<T extends Packet | Message>(input: T): T {
return input;
}
export class Transformer extends Parser {
private readonly encoder: Encoder;
constructor(sourceStream: BitStream, targetStream: BitStream) {
super(sourceStream);
this.encoder = new Encoder(targetStream);
}
public transform(packetTransform: PacketTransform, messageTransform: MessageTransform) {
this.encoder.encodeHeader(this.getHeader());
for (const message of this.getMessages()) {
this.parserState.handleMessage(message);
if (message.type === MessageType.Packet) {
for (const packet of message.packets) {
this.parserState.handlePacket(packet);
}
message.packets = message.packets.map(packetTransform);
}
this.encoder.writeMessage(messageTransform(message));
}
}
}

BIN
src/tests/data/short.dem Normal file

Binary file not shown.

695
src/tests/data/short.json Normal file
View file

@ -0,0 +1,695 @@
{
"chat": [
{
"kind": "TF_Chat_All",
"from": "-[MG]- Linc",
"text": "hf",
"tick": 41683
},
{
"kind": "TF_Chat_All",
"from": "-[MG]- Linc",
"text": "gg",
"tick": 74952
},
{
"kind": "TF_Chat_All",
"from": "sientalprime",
"text": "gr",
"tick": 75279
}
],
"users": {
"2": {
"classes": {},
"name": "SourceTV",
"steamId": "BOT",
"userId": 2
},
"3": {
"classes": {
"3": 12
},
"name": "Oneflower",
"steamId": "[U:1:72084]",
"userId": 3,
"team": "blue"
},
"4": {
"classes": {
"4": 10,
"5": 2
},
"name": "THG | The Bearded MIG",
"steamId": "[U:1:79943218]",
"userId": 4,
"team": "blue"
},
"5": {
"classes": {
"4": 3,
"5": 7,
"6": 1
},
"name": "sientalprime",
"steamId": "[U:1:315192136]",
"userId": 5,
"team": "blue"
},
"6": {
"classes": {
"1": 11,
"2": 2,
"6": 1,
"7": 4
},
"name": "Colonel Miggy-Bears",
"steamId": "[U:1:4783268]",
"userId": 6,
"team": "blue"
},
"7": {
"classes": {
"1": 11,
"2": 1
},
"name": "-[MG]- Linc",
"steamId": "[U:1:25900190]",
"userId": 7,
"team": "blue"
},
"8": {
"classes": {
"3": 11
},
"name": "J1ll ✧ stipuha zavisimyy",
"steamId": "[U:1:70354528]",
"userId": 8,
"team": "red"
},
"9": {
"classes": {
"3": 10
},
"name": "dzapis",
"steamId": "[U:1:134795814]",
"userId": 9,
"team": "red"
},
"10": {
"classes": {
"1": 1,
"3": 10,
"8": 2
},
"name": "SAVE_THE_SPYCRABS",
"steamId": "[U:1:125755589]",
"userId": 10,
"team": "blue"
},
"11": {
"classes": {
"1": 9
},
"name": "skq",
"steamId": "[U:1:113575586]",
"userId": 11,
"team": "red"
},
"12": {
"classes": {
"4": 7
},
"name": "Dunkelheit",
"steamId": "[U:1:52631210]",
"userId": 12,
"team": "red"
},
"13": {
"classes": {
"1": 7
},
"name": "Alexander",
"steamId": "[U:1:101341390]",
"userId": 13,
"team": "red"
},
"14": {
"classes": {
"5": 6
},
"name": "Dr.Oetker",
"steamId": "[U:1:76315857]",
"userId": 14,
"team": "red"
}
},
"deaths": [
{
"killer": 3,
"assister": null,
"victim": 11,
"weapon": "quake_rl",
"tick": 41352
},
{
"killer": 11,
"assister": null,
"victim": 6,
"weapon": "scattergun",
"tick": 43136
},
{
"killer": 9,
"assister": 12,
"victim": 7,
"weapon": "quake_rl",
"tick": 43190
},
{
"killer": 11,
"assister": 12,
"victim": 10,
"weapon": "scattergun",
"tick": 43515
},
{
"killer": 11,
"assister": null,
"victim": 3,
"weapon": "scattergun",
"tick": 43817
},
{
"killer": 8,
"assister": 14,
"victim": 4,
"weapon": "shotgun_soldier",
"tick": 44729
},
{
"killer": 12,
"assister": 14,
"victim": 7,
"weapon": "tf_projectile_pipe_remote",
"tick": 45866
},
{
"killer": 3,
"assister": 7,
"victim": 8,
"weapon": "world",
"tick": 45900
},
{
"killer": 12,
"assister": 14,
"victim": 3,
"weapon": "tf_projectile_pipe_remote",
"tick": 45954
},
{
"killer": 12,
"assister": 14,
"victim": 6,
"weapon": "tf_projectile_pipe_remote",
"tick": 46185
},
{
"killer": 4,
"assister": null,
"victim": 13,
"weapon": "tf_projectile_pipe",
"tick": 46445
},
{
"killer": 11,
"assister": 12,
"victim": 10,
"weapon": "scattergun",
"tick": 46456
},
{
"killer": 9,
"assister": null,
"victim": 4,
"weapon": "quake_rl",
"tick": 46491
},
{
"killer": 12,
"assister": null,
"victim": 5,
"weapon": "tf_projectile_pipe",
"tick": 46607
},
{
"killer": 11,
"assister": 9,
"victim": 7,
"weapon": "scattergun",
"tick": 48554
},
{
"killer": 8,
"assister": 13,
"victim": 4,
"weapon": "cow_mangler",
"tick": 48693
},
{
"killer": 11,
"assister": 12,
"victim": 6,
"weapon": "scattergun",
"tick": 48763
},
{
"killer": 9,
"assister": 8,
"victim": 3,
"weapon": "quake_rl",
"tick": 48835
},
{
"killer": 13,
"assister": 14,
"victim": 10,
"weapon": "scattergun",
"tick": 48959
},
{
"killer": 11,
"assister": null,
"victim": 5,
"weapon": "scattergun",
"tick": 49312
},
{
"killer": 8,
"assister": null,
"victim": 4,
"weapon": "shotgun_soldier",
"tick": 51581
},
{
"killer": 10,
"assister": 3,
"victim": 8,
"weapon": "tf_projectile_rocket",
"tick": 51658
},
{
"killer": 12,
"assister": null,
"victim": 3,
"weapon": "tf_projectile_pipe_remote",
"tick": 51866
},
{
"killer": 12,
"assister": null,
"victim": 7,
"weapon": "tf_projectile_pipe_remote",
"tick": 51907
},
{
"killer": 6,
"assister": null,
"victim": 11,
"weapon": "degreaser",
"tick": 52054
},
{
"killer": 12,
"assister": 13,
"victim": 6,
"weapon": "tf_projectile_pipe",
"tick": 52087
},
{
"killer": 10,
"assister": null,
"victim": 12,
"weapon": "tf_projectile_rocket",
"tick": 52117
},
{
"killer": 9,
"assister": null,
"victim": 10,
"weapon": "quake_rl",
"tick": 52431
},
{
"killer": 5,
"assister": null,
"victim": 9,
"weapon": "ubersaw",
"tick": 52628
},
{
"killer": 14,
"assister": 13,
"victim": 5,
"weapon": "ubersaw",
"tick": 52721
},
{
"killer": 9,
"assister": 11,
"victim": 4,
"weapon": "quake_rl",
"tick": 55162
},
{
"killer": 3,
"assister": 4,
"victim": 9,
"weapon": "quake_rl",
"tick": 55369
},
{
"killer": 13,
"assister": 11,
"victim": 7,
"weapon": "pistol_scout",
"tick": 55645
},
{
"killer": 12,
"assister": 14,
"victim": 3,
"weapon": "tf_projectile_pipe",
"tick": 56902
},
{
"killer": 10,
"assister": 5,
"victim": 8,
"weapon": "tf_projectile_rocket",
"tick": 57415
},
{
"killer": 12,
"assister": null,
"victim": 5,
"weapon": "tf_projectile_pipe_remote",
"tick": 57785
},
{
"killer": 12,
"assister": 14,
"victim": 10,
"weapon": "tf_projectile_pipe",
"tick": 57963
},
{
"killer": 7,
"assister": null,
"victim": 13,
"weapon": "force_a_nature",
"tick": 58048
},
{
"killer": 4,
"assister": 7,
"victim": 11,
"weapon": "tf_projectile_pipe_remote",
"tick": 58164
},
{
"killer": 9,
"assister": 12,
"victim": 6,
"weapon": "quake_rl",
"tick": 58395
},
{
"killer": 7,
"assister": 3,
"victim": 12,
"weapon": "force_a_nature",
"tick": 58756
},
{
"killer": 9,
"assister": 14,
"victim": 7,
"weapon": "quake_rl",
"tick": 58758
},
{
"killer": 9,
"assister": null,
"victim": 3,
"weapon": "quake_rl",
"tick": 59038
},
{
"killer": 14,
"assister": 9,
"victim": 4,
"weapon": "ubersaw",
"tick": 59157
},
{
"killer": 9,
"assister": 11,
"victim": 3,
"weapon": "quake_rl",
"tick": 61550
},
{
"killer": 12,
"assister": 8,
"victim": 4,
"weapon": "tf_projectile_pipe_remote",
"tick": 61819
},
{
"killer": 9,
"assister": 8,
"victim": 10,
"weapon": "quake_rl",
"tick": 62325
},
{
"killer": 13,
"assister": null,
"victim": 7,
"weapon": "pistol_scout",
"tick": 62772
},
{
"killer": 6,
"assister": null,
"victim": 8,
"weapon": "sniperrifle",
"tick": 63063
},
{
"killer": 5,
"assister": null,
"victim": 9,
"weapon": "ubersaw",
"tick": 63337
},
{
"killer": 11,
"assister": null,
"victim": 6,
"weapon": "scattergun",
"tick": 63853
},
{
"killer": 11,
"assister": 13,
"victim": 7,
"weapon": "scattergun",
"tick": 64916
},
{
"killer": 13,
"assister": null,
"victim": 10,
"weapon": "world",
"tick": 65203
},
{
"killer": 12,
"assister": null,
"victim": 3,
"weapon": "tf_projectile_pipe_remote",
"tick": 65305
},
{
"killer": 11,
"assister": 8,
"victim": 6,
"weapon": "scattergun",
"tick": 66212
},
{
"killer": 12,
"assister": 13,
"victim": 5,
"weapon": "tf_projectile_pipe_remote",
"tick": 66256
},
{
"killer": 9,
"assister": 14,
"victim": 4,
"weapon": "quake_rl",
"tick": 66343
},
{
"killer": 10,
"assister": null,
"victim": 11,
"weapon": "tf_projectile_rocket",
"tick": 68654
},
{
"killer": 13,
"assister": null,
"victim": 10,
"weapon": "scattergun",
"tick": 68695
},
{
"killer": 9,
"assister": 13,
"victim": 6,
"weapon": "quake_rl",
"tick": 68952
},
{
"killer": 9,
"assister": 8,
"victim": 3,
"weapon": "quake_rl",
"tick": 69063
},
{
"killer": 5,
"assister": null,
"victim": 12,
"weapon": "iron_bomber",
"tick": 69227
},
{
"killer": 5,
"assister": null,
"victim": 9,
"weapon": "iron_bomber",
"tick": 71107
},
{
"killer": 12,
"assister": 14,
"victim": 6,
"weapon": "tf_projectile_pipe",
"tick": 72276
},
{
"killer": 7,
"assister": 3,
"victim": 8,
"weapon": "force_a_nature",
"tick": 73216
},
{
"killer": 12,
"assister": null,
"victim": 7,
"weapon": "tf_projectile_pipe",
"tick": 73641
},
{
"killer": 3,
"assister": null,
"victim": 11,
"weapon": "quake_rl",
"tick": 73797
},
{
"killer": 5,
"assister": null,
"victim": 9,
"weapon": "tf_projectile_pipe_remote",
"tick": 74070
},
{
"killer": 13,
"assister": null,
"victim": 3,
"weapon": "scattergun",
"tick": 74284
},
{
"killer": 14,
"assister": 12,
"victim": 4,
"weapon": "crusaders_crossbow",
"tick": 74355
},
{
"killer": 12,
"assister": null,
"victim": 5,
"weapon": "tf_projectile_pipe",
"tick": 74639
},
{
"killer": 12,
"assister": null,
"victim": 10,
"weapon": "tf_projectile_pipe",
"tick": 74639
},
{
"killer": 13,
"assister": 12,
"victim": 6,
"weapon": "scattergun",
"tick": 74679
},
{
"assister": null,
"victim": 8,
"weapon": "worldspawn",
"tick": 75044
}
],
"rounds": [
{
"winner": "red",
"length": 73.81500244140625,
"end_tick": 46460
},
{
"winner": "red",
"length": 86.14501953125,
"end_tick": 52870
},
{
"winner": "red",
"length": 85.76995849609375,
"end_tick": 59255
},
{
"winner": "red",
"length": 97.02001953125,
"end_tick": 66390
},
{
"winner": "red",
"length": 117.405029296875,
"end_tick": 74884
}
],
"startTick": 41206,
"intervalPerTick": 0.014999999664723873
}

View file

@ -0,0 +1,39 @@
import * as assert from 'assert';
import {BitStream} from 'bit-buffer';
import {readFileSync} from 'fs';
import {DynamicBitStream} from '../../DynamicBitStream';
import {nullTransform, Transformer} from '../../Transformer';
import {Parser} from '../../Parser';
import {Analyser} from '../../Analyser';
function testDemo(name: string) {
const target = JSON.parse(readFileSync(`${__dirname}/../data/${name}.json`, 'utf8'));
const decodeStream = new BitStream(
readFileSync(`${__dirname}/../data/${name}.dem`).buffer as ArrayBuffer
);
const encodeStream = new DynamicBitStream(32 * 1024 * 1024);
const transformer = new Transformer(decodeStream, encodeStream);
transformer.transform(nullTransform, nullTransform);
const encodedLength = encodeStream.index;
encodeStream.index = 0;
console.log('start reparse');
const reParser = new Parser(encodeStream);
const analyser = new Analyser(reParser);
const parsed = analyser.getBody().getState();
const reParsedLength = encodeStream.index;
assert.equal(reParsedLength, encodedLength, 'Unexpected number of bits used when parsing encoding stream');
assert.deepEqual(JSON.parse(JSON.stringify(parsed)), target);
}
suite('Transcode demo', () => {
test('Noop transcode', () => {
testDemo('short');
});
});